CUNY 2020 Analysis
Packages and utilities
library(tidyverse)
library(lme4)
library(lmerTest)
library(logging)
library(mvtnorm)
library(mgcv)# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
predictions <- predict(lm, test_X, re.form=NA)
# Get std.dev. of residual, estimated from train data
stdev <- sigma(lm)
# For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
predictions <- predict(lm, test_X, re.form=NA)
# Get std.dev. of residual, estimated from train data
stdev <- sigma(lm)
# For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
densities <- dnorm(test_y, predictions, stdev, log=TRUE)
return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)Data loading and preprocessing
data = read.csv("../data/harmonized_results.csv")
all_data = data %>%
mutate(seed = as.factor(seed)) %>%
group_by(corpus, model, training, seed) %>%
mutate(prev_surp = lag(surprisal),
prev_code = lag(code),
prev_len = lag(len),
prev_freq = lag(freq),
prev_surp = lag(surprisal),
prev2_freq = lag(prev_freq),
prev2_code = lag(prev_code),
prev2_len = lag(prev_len),
prev2_surp = lag(prev_surp),
prev3_freq = lag(prev2_freq),
prev3_code = lag(prev2_code),
prev3_len = lag(prev2_len),
prev3_surp = lag(prev2_surp),
prev4_freq = lag(prev3_freq),
prev4_code = lag(prev3_code),
prev4_len = lag(prev3_len),
prev4_surp = lag(prev3_surp)) %>%
ungroup() %>%
# Filter back two for the dundee corpus. Filter back 1 for all other corpora
# NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev4_code + 4)) %>%
select(-prev_code, -prev2_code, -prev3_code) %>%
drop_na()
all_data = all_data %>%
mutate(
model = as.character(model),
model = if_else(model == "gpt-2", "gpt2", model),
model = as.factor(model))missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>%
group_by(corpus, code) %>%
filter(sum(is.na(surprisal)) > 0) %>%
ungroup() %>%
anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))
missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) + geom_bar(position=position_dodge(width=0.8))
print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n())) %>% arrange(desc(n))# Compute the ideal number of model--seed--training observations per token.
to_drop = all_data %>%
group_by(corpus, code) %>% summarise(n = n()) %>% ungroup() %>%
group_by(corpus) %>% mutate( max_n = max(n)) %>% ungroup() %>%
filter(max_n != n) %>%
select(code, corpus)
#to_drop = all_data %>% group_by(corpus, code) %>% filter(n() != ideal_token_obs_count) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))[0m2020-05-25 17:41:25 INFO::Dropping 10342 observations corresponding to corpus tokens which are missing observations for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))[0m2020-05-25 17:41:25 INFO::Dropping 10342 tokens which are missing observations for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-25 17:41:26 INFO::After drop, 962274 observations ( 33117 tokens) remain.[0m[0m[0mto_drop_zero_surps = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_surps), "observations corresponding to corpus tokens which have surprisal zeros for some model."))[0m2020-05-25 17:41:26 INFO::Dropping 116 observations corresponding to corpus tokens which have surprisal zeros for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop_zero_surps %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))[0m2020-05-25 17:41:26 INFO::Dropping 4 tokens which have surprisal zeros for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop_zero_surps %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-25 17:41:27 INFO::After drop, 962158 observations ( 33113 tokens) remain.[0m[0m[0mto_drop_zero_psychs = all_data %>% group_by(corpus, code) %>% filter(any(psychometric == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_psychs), "observations corresponding to corpus tokens which have psychometric zeros for some model."))[0m2020-05-25 17:41:27 INFO::Dropping 14935 observations corresponding to corpus tokens which have psychometric zeros for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop_zero_psychs %>% group_by(corpus, code) %>% n_groups(), "tokens which have psychometric zeros for some model."))[0m2020-05-25 17:41:27 INFO::Dropping 515 tokens which have psychometric zeros for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop_zero_psychs %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-25 17:41:27 INFO::After drop, 947223 observations ( 32598 tokens) remain.[0m[0m[0mLearn models
# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
#this_lm <- gam(formula, data=df);
this_lm = lm(formula, data=df)
this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
# Save lm to the global env so that we can access residuals later.
lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
assign(lm_name, this_lm, envir=store_env)
summarise(df,
log_lik = as.numeric(logLik(this_lm, REML = F)),
test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
# Retrieve the relevant lm.
lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
this_lm <- get(lm_name, envir=store_env)
mutate(df,
likelihood = logLik_test_per(this_lm, df, df$psychometric),
resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
baseline_lm <- get(lm_name, envir=baseline_env)
full_lm <- get(lm_name, envir=full_env)
delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.
# Eye-tracking regression: only use surprisal and previous surprisal; SPRT regression: use 2-back features.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")
#full_rt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_sprt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + s(prev3_surp, bs = "cr", k = 20) + s(prev4_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")
baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + len + prev_len + prev2_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len
full_sprt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + prev4_surp + freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len
full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + freq + prev_freq + prev2_freq + len + prev_len + prev2_len
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()
#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 10
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation
# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
if(corpus == "dundee") {
get_lm_data(df, test_data, baseline_rt_regression, fold, env)
} else {
get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
}
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
if(corpus[1] == "dundee") {
get_lm_data(df, test_data, full_rt_regression, fold, env)
} else {
get_lm_data(df, test_data, full_sprt_regression, fold, env)
}
}
# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()
for(i in 1:K) {
#Segement your data by fold using the which() function
testIndexes <- which(folds==i, arr.ind=TRUE)
test_data <- all_data[testIndexes, ]
train_data <- all_data[-testIndexes, ]
# Compute a baseline linear model for each model--training--seed--RT-corpus combination.
baselines = train_data %>%
group_by(model, training, seed, corpus) %>%
print(model) %>%
do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
ungroup() %>%
mutate(seed = as.factor(seed),
fold = i)
baseline_results = rbind(baseline_results, baselines)
# Compute a full linear model for each model--training--seed-RT-corpus combination
full_models = train_data %>%
group_by(model, training, seed, corpus) %>%
do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
ungroup() %>%
mutate(seed = as.factor(seed),
fold = i)
full_model_results = rbind(full_model_results, full_models)
# Compute delta-log-likelihoods
fold_log_lik_deltas = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
ungroup()
log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
fold_baseline_residuals = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_residuals(., i, baseline_env)) %>%
ungroup()
baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)
fold_full_residuals = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_residuals(., i, full_env)) %>%
ungroup()
full_residuals = rbind(full_residuals, fold_full_residuals)
}
|===================================================================================================================================================== | 68% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================== | 43% ~3 s remaining
|====================================================================================================== | 47% ~3 s remaining
|============================================================================================================= | 50% ~3 s remaining
|==================================================================================================================== | 53% ~2 s remaining
|============================================================================================================================ | 57% ~2 s remaining
|=================================================================================================================================== | 60% ~2 s remaining
|=========================================================================================================================================== | 64% ~2 s remaining
|================================================================================================================================================== | 67% ~2 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~1 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================================================ | 72% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|==================================================================================================================================================================== | 75% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|================================================================================================================================================================================ | 81% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================================================== | 84% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|=============================================================================================================================================================================================== | 88% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|======================================================================================================================================================================================================= | 91% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|====================================================================================================================================================================================================================== | 98% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================================================================== | 80% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|==================================================================================================================================================================== | 75% ~1 s remaining
|====================================================================================================================================================================== | 76% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|============================================================================================================================================================================== | 80% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================================================================== | 80% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================================================================== | 80% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|====================================================================================================================================================================================================================== | 98% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================================= | 65% ~1 s remaining
|================================================================================================================================================== | 67% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|====================================================================================================================================================================== | 76% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|============================================================================================================================ | 57% ~2 s remaining
|=================================================================================================================================== | 60% ~1 s remaining
|=========================================================================================================================================== | 64% ~1 s remaining
|================================================================================================================================================== | 67% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~1 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|====================================================================================================================================== | 61% ~1 s remaining
|=========================================================================================================================================== | 64% ~1 s remaining
|================================================================================================================================================== | 67% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|============================================================================================================================================================ | 72% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|===========================================================================================================================================================================================================================|100% ~0 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|====================================================================================================================================================================================================================== | 98% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|===========================================================================================================================================================================================================================|100% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|==================================================================================================================== | 53% ~2 s remaining
|============================================================================================================================ | 57% ~2 s remaining
|=================================================================================================================================== | 60% ~1 s remaining
|=========================================================================================================================================== | 64% ~1 s remaining
|================================================================================================================================================== | 67% ~1 s remaining
|======================================================================================================================================================= | 69% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|========================================================================================================================================================================= | 77% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~1 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|=========================================================================================================== | 49% ~2 s remaining
|============================================================================================================= | 50% ~2 s remaining
|==================================================================================================================== | 53% ~2 s remaining
|============================================================================================================================ | 57% ~2 s remaining
|=================================================================================================================================== | 60% ~2 s remaining
|=========================================================================================================================================== | 64% ~1 s remaining
|================================================================================================================================================== | 67% ~1 s remaining
|========================================================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~1 s remaining
|========================================================================================================================================================================================== | 85% ~1 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining
|================================================================================================================================================================= | 74% ~1 s remaining
|=========================================================================================================================================================================== | 78% ~1 s remaining
|=================================================================================================================================================================================== | 82% ~0 s remaining
|========================================================================================================================================================================================== | 85% ~0 s remaining
|================================================================================================================================================================================================== | 89% ~0 s remaining
|========================================================================================================================================================================================================= | 92% ~0 s remaining
|================================================================================================================================================================================================================= | 95% ~0 s remaining
|======================================================================================================================================================================================================================== | 99% ~0 s remaining #write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")model_deltas = log_lik_deltas %>%
group_by(model, training, seed, corpus) %>%
summarise(mean_delta_log_lik = mean(delta_log_lik),
sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")metric <- "ΔLogLik"
#metric <- "-ΔMSE"
# # Select the relevant metric.
model_deltas = model_deltas %>%
# Retrieve the current test metric
mutate(delta_test_mean = mean_delta_log_lik,
delta_test_sem = sem_delta_log_lik) %>%
# mutate(delta_test_mean = mean_delta_mse,
# delta_test_sem = sem_delta_mse)
# Remove the raw metrics.
select(-mean_delta_log_lik, -sem_delta_log_lik,
#-mean_delta_mse, -sem_delta_mse
)
model_deltas# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
# full_baselines = all_data %>%
# group_by(model, training, seed, corpus) %>%
# summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
# ungroup()
# full_baselines
#
# full_baselines %>%
# right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
# mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
# select(-baseline_lik) # %>%
# #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)Load language model data (SyntaxGym, PPL)
language_model_data = read.csv("../data/model_metadata.csv") %>%
mutate(model = as.character(model),
model = if_else(model == "gpt-2", "gpt2", model),
model = as.factor(model)) %>%
mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
str_starts(training, "bllip-md") ~ 15,
str_starts(training, "bllip-sm") ~ 5,
str_starts(training, "bllip-xs") ~ 1),
# Training vocabulary usually covaries with the training corpus.
# But BPE models share a vocabulary across training corpora.
training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))
) %>%
mutate(seed = as.factor(seed)) %>%
select(-pid, -test_loss) %>%
distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)
0 111 120 922 1111 3602 4301 7245 7877 28066 28068 44862 51272 64924 1581807512 1581807578 1581861474 1581955288 1582126320 1586986276 1587139950
4 7 6 5 4 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 table(model_deltas$seed)
111 120 607 922 1111 3602 4301 7245 7877 28066 28068 44862 51272 64924 1581807512 1581807578 1581861474 1581955288 1582126320 1586986276 1587139950
9 9 1 9 12 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 First join delta-metric data with model auxiliary data.
model_deltas = model_deltas %>%
merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
drop_na()
model_deltasAlso join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.
Final data preprocessing
# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
filter(model != "ordered-neurons")Visualizations
The basics
all_data %>% ggplot(aes(x=corpus)) + geom_bar()
print(all_data %>% group_by(corpus) %>% summarise(n=n()))all_data %>%
ggplot(aes(x=freq, color=corpus)) + geom_density()
all_data %>%
ggplot(aes(x=len, color=corpus)) + geom_density()
all_data %>%
ggplot(aes(x=surprisal, color=corpus)) + geom_density()
Predictive power and SG
model_deltas %>%
ggplot(aes(x=sg_score, y=delta_test_mean)) +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
geom_smooth(method="lm", se=T) +
geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
ylab(metric) +
xlab("Syntax Generalization Score") +
ggtitle("Syntactic Generalization vs. Predictive Power") +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"gptbpe"="#888888")) +
facet_grid(~corpus, scales="free") +
theme(axis.text=element_text(size=14),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "bottom")
#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)Regression analyses
We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.
do_stepwise_regression = function(cur_corpus) {
regression_data = model_deltas %>%
filter(corpus == cur_corpus)
print("----------------------")
print(cur_corpus)
lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
print(anova(lm1, lm2))
summary(lm2)
}
do_stepwise_regression("bnc-brown")[1] "----------------------"
[1] "bnc-brown"
Analysis of Variance Table
Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
Res.Df RSS Df Sum of Sq F Pr(>F)
1 23 0.00022595
2 22 0.00022153 1 0.0000044212 0.4391 0.5145
Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score,
data = regression_data)
Residuals:
Min 1Q Median 3Q Max
-0.0047409 -0.0020020 -0.0006643 0.0007764 0.0065040
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 0.012686214 0.003485419 3.640 0.00145 **
sg_score 0.003288759 0.004963235 0.663 0.51445
training_vocabbllip-lg:test_ppl 0.000021841 0.000032293 0.676 0.50587
training_vocabbllip-md:test_ppl -0.000036599 0.000026912 -1.360 0.18762
training_vocabbllip-sm:test_ppl -0.000047892 0.000023965 -1.998 0.05818 .
training_vocabbllip-xs:test_ppl -0.000057886 0.000015989 -3.620 0.00152 **
training_vocabgptbpe:test_ppl -0.000009744 0.000007856 -1.240 0.22793
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.003173 on 22 degrees of freedom
Multiple R-squared: 0.6354, Adjusted R-squared: 0.536
F-statistic: 6.39 on 6 and 22 DF, p-value: 0.0005242do_stepwise_regression("dundee")[1] "----------------------"
[1] "dundee"
Analysis of Variance Table
Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
Res.Df RSS Df Sum of Sq F Pr(>F)
1 23 0.000096017
2 22 0.000095653 1 0.00000036375 0.0837 0.7751
Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score,
data = regression_data)
Residuals:
Min 1Q Median 3Q Max
-0.0025202 -0.0011339 -0.0007746 0.0011467 0.0038433
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 0.007166240 0.002290290 3.129 0.00488 **
sg_score 0.000943334 0.003261371 0.289 0.77510
training_vocabbllip-lg:test_ppl -0.000011756 0.000021220 -0.554 0.58517
training_vocabbllip-md:test_ppl -0.000017966 0.000017684 -1.016 0.32071
training_vocabbllip-sm:test_ppl -0.000023214 0.000015748 -1.474 0.15460
training_vocabbllip-xs:test_ppl -0.000022899 0.000010507 -2.179 0.04029 *
training_vocabgptbpe:test_ppl -0.000001882 0.000005162 -0.364 0.71898
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.002085 on 22 degrees of freedom
Multiple R-squared: 0.3224, Adjusted R-squared: 0.1376
F-statistic: 1.744 on 6 and 22 DF, p-value: 0.1578do_stepwise_regression("natural-stories")[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table
Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
Res.Df RSS Df Sum of Sq F Pr(>F)
1 23 0.000043247
2 22 0.000039544 1 0.0000037027 2.06 0.1653
Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score,
data = regression_data)
Residuals:
Min 1Q Median 3Q Max
-0.0027593 -0.0006678 0.0002825 0.0007265 0.0020339
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 0.011076788 0.001472587 7.522 0.000000162 ***
sg_score -0.003009675 0.002096963 -1.435 0.1653
training_vocabbllip-lg:test_ppl -0.000069216 0.000013644 -5.073 0.000044141 ***
training_vocabbllip-md:test_ppl -0.000060246 0.000011370 -5.299 0.000025649 ***
training_vocabbllip-sm:test_ppl -0.000068670 0.000010125 -6.782 0.000000816 ***
training_vocabbllip-xs:test_ppl -0.000040089 0.000006755 -5.934 0.000005680 ***
training_vocabgptbpe:test_ppl -0.000007666 0.000003319 -2.310 0.0307 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.001341 on 22 degrees of freedom
Multiple R-squared: 0.7617, Adjusted R-squared: 0.6967
F-statistic: 11.72 on 6 and 22 DF, p-value: 0.000006701Predictive power and perplexity
model_deltas %>%
mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl)) %>%
ggplot(aes(x=test_ppl, y=delta_test_mean, color=training_vocab, fill = training_vocab, ymin=0)) +
theme_bw() +
geom_text(aes(x=275, y=0, label = c("//"))) +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), alpha=0.4) +
#geom_smooth(method="lm", se=F) +
geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color = training_vocab)) +
ylab(metric) +
xlab("Test Perplexity") +
#coord_cartesian(ylim = c(1, 16)) +
ggtitle("Test Perplexity vs. Predictive Power") +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"gptbpe"="#888888")) +
scale_shape_manual(values = c(16, 17, 15, 18)) +
scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, " ", 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300,305, 350), minor_breaks = NULL) +
scale_y_continuous(limits = c(0, NA), expand = c(0,0)) +
facet_wrap(~corpus, scales="free") +
coord_cartesian(clip="off") +
theme(axis.text=element_text(size=12),
strip.text.x = element_text(size=12),
legend.text=element_text(size=12),
axis.title=element_text(size=12),
legend.position = "right")
ggsave("../images/cuny2020/ppl_loglik.png",height=4.5,width=11)
dll_cor_test = function(df){
df %>%
summarise(
cor = cor.test(df$delta_test_mean, df$test_ppl)$estimate,
p = cor.test(df$delta_test_mean, df$test_ppl)$p.value
)
}
model_deltas %>%
filter(model != "5gram") %>%
group_by(training, corpus) %>%
mutate(n = n()) %>%
ungroup() %>%
filter(n > 2) %>%
group_by(training, corpus) %>%
do({ dll_cor_test(.) }) %>%
ungroup() %>%
arrange(corpus)Effect of training data size
model_deltas %>%
mutate(train_size = log(train_size)) %>%
mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
bpe = as.factor(bpe)) %>%
ggplot(aes(x=train_size, y=delta_test_mean, color=model)) +
theme_bw() +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), width = 0.1) +
geom_smooth(method="lm", se=T, alpha=0.2) +
geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(shape=bpe)) +
ylab(metric) +
xlab("Log Million Training Tokens") +
ggtitle("Training Size vs. Predictive Power") +
facet_grid(.~corpus, scales="free") +
#scale_color_manual(values = c("#A42EF1", "#3894C8")) +
theme(axis.text=element_text(size=12),
strip.text.x = element_text(size=12),
legend.text=element_text(size=8),
legend.title=element_text(size=8),
axis.title=element_text(size=14),
legend.position = "bottom",
legend.direction = "horizontal",
legend.key.width = unit(0.3,"cm"),
legend.spacing.x = unit(0.1, 'cm'))
ggsave("../images/cuny2020/training_loglik.png",height=5,width=5)
model_cor_test = function(df){
df %>%
summarise(cor = cor.test(df$train_size, df$delta_test_mean)$estimate,
p = cor.test(df$train_size, df$delta_test_mean)$p.value)
}
model_deltas %>%
group_by(model, corpus) %>%
do({model_cor_test(.)}) %>%
ungroup() %>%
arrange()model_deltas %>%
mutate(train_size = log(train_size)) %>%
mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
bpe = as.factor(bpe)) %>%
ggplot(aes(x=train_size, y=sg_score, color=model)) +
theme_bw() +
geom_smooth(method="lm", se=T, alpha=0.2) +
geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(shape=bpe)) +
ylab("SG SCore") +
xlab("Log Million Training Tokens") +
ggtitle("Training Size vs. SG Score") +
#scale_color_manual(values = c("#A42EF1", "#3894C8")) +
#facet_grid(~model, scales="free") +
theme(axis.text=element_text(size=12),
strip.text.x = element_text(size=12),
legend.text=element_text(size=8),
legend.title=element_text(size=8),
axis.title=element_text(size=14),
legend.position = "bottom",
legend.direction = "horizontal",
legend.key.width = unit(0.3,"cm"),
legend.spacing.x = unit(0.1, 'cm'))
ggsave("../images/cuny2020/training_sg.png",height=5,width=4)
model_cor_test = function(df){
df %>%
summarise(cor = cor.test(df$train_size, df$sg_score)$estimate,
p = cor.test(df$train_size, df$sg_score)$p.value)
}
model_deltas %>%
group_by(model) %>%
do({model_cor_test(.)}) %>%
ungroup()Smith & Levy reproduction
all_data %>%
ggplot(aes(x=surprisal)) +
theme_bw() +
geom_density() +
facet_grid(~corpus) +
coord_cartesian(xlim = c(0, 21)) +
theme(panel.spacing = unit(2.5, "cm"))
ggsave("../images/cuny2020/surp_corr_marginals.png",height=1.5,width=11)
all_data %>%
#filter(model == "gpt2") %>%
#filter(surprisal < 15, surprisal > 0) %>%
filter(surprisal<21) %>%
mutate(bpe=str_detect(training, "bpe"),
training_source=str_replace(training, "-gptbpe", "")) %>%
ggplot(aes(x=surprisal, y=psychometric, color=training_source, linetype=bpe)) +
theme_bw() +
stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
#geom_point(stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration") +
facet_wrap(model ~ corpus, scales="free", ncol=3, strip.position = c("right")) +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"bllip-lg-gptbpe"="#888888",
"bllip-md-gptbpe"="#888888",
"bllip-sm-gptbpe"="#888888",
"bllip-xs-gptbpe"="#888888")) +
coord_cartesian(xlim = c(0, 21)) +
theme(axis.text=element_text(size=10),
axis.text.y = element_text(size = 10),
strip.text.x = element_text(size=10),
legend.text=element_text(size=10),
axis.title=element_text(size=12),
legend.position = "right")
ggsave("../images/cuny2020/surp_corr.png",height=6,width=12)
corr_test = function(df){
df %>%
summarise(
cor = cor.test(df$surprisal, df$psychometric)$estimate
)
}
all_data %>%
group_by(model, training, corpus, seed) %>%
do({ cor = corr_test(.)}) %>%
ungroup()
Investigate vanilla
all_data %>%
#filter(surprisal < 15, surprisal > 0) %>%
filter(model == "vanilla") %>%
ggplot(aes(x=surprisal, y=psychometric)) +
#stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration: Vanilla") +
facet_grid(corpus~training, scales = "free")
# scale_color_manual(values = c("bllip-lg"="#440154FF",
# "bllip-md"="#39568CFF",
# "bllip-sm"="#1F968BFF",
# "bllip-xs"="#73D055FF",
# "bllip-lg-gptbpe"="#888888",
# "bllip-md-gptbpe"="#888888",
# "bllip-sm-gptbpe"="#888888",
# "bllip-xs-gptbpe"="#888888"))all_data %>%
filter(corpus == "dundee", model == "vanilla", training == "bllip-lg", surprisal > 20, psychometric < 300)print(full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)))
full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>%
ggplot(aes(x=surprisal)) + geom_density()Investigate RNNG
all_data %>%
#filter(surprisal < 15, surprisal > 0) %>%
filter(model == "rnng") %>%
ggplot(aes(x=surprisal, y=psychometric)) +
#stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration: RNNG") +
facet_grid(corpus~training, scales = "free")all_data %>%
filter(corpus == "dundee", model == "rnng", training == "bllip-lg", surprisal > 20, psychometric < 300)print(full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)))
full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>%
ggplot(aes(x=surprisal)) + geom_density()Investigate ngram vs vanilla
ngram_resids = full_residuals %>% filter(model == "5gram", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal), resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal), resid=mean(resid))
resids_joined = ngram_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".ngram", ".vanilla"))
resids_joined %>%
ggplot(aes(x=resid.ngram, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_diff=resid.ngram - resid.vanilla) %>%
ggplot(aes(x=resid_diff)) + geom_density() +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_diff=abs(resid.ngram) - abs(resid.vanilla),
big=resid_diff < -10) %>%
ggplot(aes(x=surprisal.ngram, color=big)) + geom_density() + facet_grid(~corpus) +
ggtitle("ngram surprisal of high-improvement tokens (relative to vanilla)")
resids_joined %>%
mutate(resid_abs_diff=abs(resid.ngram - resid.vanilla)) %>%
ggplot(aes(x=freq.ngram, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()Investigate gptbpe vs vanilla
gpt_resids = full_residuals %>% filter(model == "gpt2", training == "bllip-sm-gptbpe") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal), resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), surprisal=mean(surprisal), resid=mean(resid))
resids_joined = gpt_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".gpt", ".vanilla"))
resids_joined %>%
ggplot(aes(x=resid.gpt, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_diff=resid.gpt - resid.vanilla) %>%
ggplot(aes(x=resid_diff)) + geom_density() +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_diff=abs(resid.gpt) - abs(resid.vanilla),
big=resid_diff < -10) %>%
ggplot(aes(x=surprisal.gpt, color=big)) + geom_density() + facet_grid(~corpus) +
ggtitle("gpt surprisal of high-improvement tokens (relative to vanilla)")
resids_joined %>%
mutate(resid_abs_diff=abs(resid.gpt - resid.vanilla)) %>%
ggplot(aes(x=freq.gpt, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()Investigate residuals overall
resid_deltas = full_residuals %>% right_join(baseline_residuals, by=c("corpus", "code", "model", "training", "seed"), suffix=c(".full", ".baseline")) %>%
select(resid.baseline, resid.full, code, surprisal.full, psychometric.full, model, training, seed, corpus, len.full) %>%
mutate(resid.baseline.pol = if_else(resid.baseline > 0, 1, 0),
resid.full.pol = if_else(resid.full > 0, 1, 0)) %>%
mutate(resid.baseline = abs(resid.baseline),
resid.full = abs(resid.full)) %>%
mutate(resid_delta=resid.baseline - resid.full, #positive is better
training_source=as.factor(str_replace(training, "-gptbpe", "")),
bpe=str_detect(training, "gptbpe"))
r = resid_deltas %>%
filter(resid.full.pol != resid.baseline.pol)
resid_deltas %>%
ggplot(aes(x=surprisal.full, y=resid_delta, color=training)) +
facet_grid(model~corpus) +
geom_point(alpha=0.1, size=0.5)language_model_data %>% filter(model == "gpt2")resid_deltas %>%
group_by(corpus) %>%
mutate(psychometric = scale(psychometric.full)) %>%
ungroup() %>%
ggplot(aes(x=psychometric)) +
theme_bw() +
geom_density() +
geom_vline(xintercept = 0, color = "grey") +
facet_grid(.~corpus) +
#coord_cartesian(xlim = c(-2, 4)) +
theme(axis.title.x=element_blank(),
axis.text.x=element_blank(),
axis.ticks.x=element_blank())
#ggsave("length.png", width = 8, height = 1)
log_lik_deltas %>%
#resid_deltas %>%
#filter(resid.full.pol == resid.baseline.pol) %>%
group_by(corpus) %>%
mutate(psychometric = scale(psychometric)) %>%
ungroup() %>%
#filter(psychometric < 4) %>%
#filter(len.full <= 10) %>%
ggplot(aes(x = psychometric, y = delta_log_lik, color = model)) +
theme_bw() +
facet_grid(. ~ corpus, scales = "free") +
#geom_rug(alpha = 0.003, sides = "b") +
geom_hline(yintercept=0, color = "blue") +
geom_vline(xintercept = 0, color = "grey") +
geom_smooth(se = T, alpha = 0.2) +
coord_cartesian(ylim = c(-0.1, 0.2), xlim = c(-2, 4)) +
theme(legend.position = "bottom",
strip.text.x = element_blank())
ggsave( "./resid_psycho.png", height = 4, width = 8)
resid_deltas %>%
group_by(corpus) %>%
mutate(psychometric = scale(psychometric.full)) %>%
ungroup() %>%
ggplot(aes(x=len.full)) +
theme_bw() +
geom_histogram(bins = 20) +
geom_vline(xintercept = 0, color = "grey") +
facet_grid(.~corpus) +
coord_cartesian(xlim = c(1, 10)) +
theme(axis.title.x=element_blank(),
axis.text.x=element_blank(),
axis.ticks.x=element_blank())
ggsave("length_maringals.png", width = 8, height = 1)
log_lik_deltas %>%
#resid_deltas %>%
#filter(resid.full.pol == resid.baseline.pol) %>%
group_by(corpus) %>%
mutate(psychometric = scale(psychometric)) %>%
ungroup() %>%
#filter(psychometric < 4) %>%
#filter(len.full <= 10) %>%
ggplot(aes(x = len, y = delta_log_lik, color = model)) +
theme_bw() +
facet_grid(. ~ corpus, scales = "free") +
#geom_rug(alpha = 0.003, sides = "b") +
geom_hline(yintercept=0, color = "blue") +
geom_vline(xintercept = 0, color = "grey") +
geom_smooth(se = T, alpha = 0.2) +
coord_cartesian(ylim = c(-0.02, 0.06), xlim = c(1, 10)) +
theme(legend.position = "bottom",
strip.text.x = element_blank())
ggsave( "./resid_length.png", height = 4, width = 8)
word_norm = log_lik_deltas %>%
drop_na() %>%
group_by(word, corpus, model, training, seed) %>%
mutate(psychoword = scale(psychometric),
norm_surp = scale(surprisal))word_norm %>%
ggplot(aes(x=norm_surp)) +
facet_grid(~corpus) +
geom_density() +
coord_cartesian(xlim = c(-2, 5)) +
geom_vline(xintercept = 0, color = "grey") +
theme_bw() +
theme(axis.title.x=element_blank(),
axis.text.x=element_blank(),
axis.ticks.x=element_blank())
ggsave("surp_maringals.png", width = 8, height = 1)
word_norm %>%
ggplot(aes(x = psychoword, y = norm_surp, color = model)) +
theme_bw() +
facet_grid(. ~ corpus, scales = "free") +
#geom_rug(alpha = 0.003, sides = "b") +
geom_hline(yintercept=0, color = "blue") +
geom_vline(xintercept = 0, color = "grey") +
geom_smooth(se = T, alpha = 0.2) +
#coord_cartesian(ylim = c(-0.05, 0.1), xlim = c(-2, 3)) +
theme(legend.position = "bottom")
#ggsave( "./resid_length.png", height = 4, width = 8)
word_norm %>%
ggplot(aes(x = norm_surp, y = delta_log_lik, color = model)) +
theme_bw() +
facet_grid( . ~ corpus, scales = "free") +
#geom_rug(alpha = 0.003, sides = "b") +
geom_hline(yintercept=0, color = "blue") +
geom_vline(xintercept = 0, color = "grey") +
geom_smooth(se = T, alpha = 0.2) +
coord_cartesian(ylim = c(-0.05, 0.07), xlim = c(-2, 5)) +
theme(legend.position = "bottom")
ggsave( "./norm_surp.png", height = 4, width = 8)
ngram_highsurp = word_norm %>%
ungroup() %>%
filter(corpus == "dundee", norm_surp > 2, model == "5gram") %>%
select(code)
ngram_highsurp = ngram_highsurp$code
z = word_norm %>%
ungroup() %>%
filter(! code %in% ngram_highsurp) %>%
filter(corpus == "dundee")
write.csv(z, "ngram-ablate.csv")
LS0tCnRpdGxlOiAiQ1VOWSAyMDIwIEFuYWx5c2lzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFBhY2thZ2VzIGFuZCB1dGlsaXRpZXMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KGxvZ2dpbmcpCmxpYnJhcnkobXZ0bm9ybSkKbGlicmFyeShtZ2N2KQpgYGAKCmBgYHtyfQojIENvbXB1dGUgdGhlIGxvZy1saWtlbGlob29kIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbG9nTGlrX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcHJlZGljdGlvbnMgPC0gcHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKQogICMgR2V0IHN0ZC5kZXYuIG9mIHJlc2lkdWFsLCBlc3RpbWF0ZWQgZnJvbSB0cmFpbiBkYXRhCiAgc3RkZXYgPC0gc2lnbWEobG0pCiAgIyBGb3IgZWFjaCBwcmVkaWN0aW9uLS1vYnNlcnZhdGlvbiwgZ2V0IHRoZSBkZW5zaXR5IHAob2JzIHwgTihwcmVkaWN0ZWQsIG1vZGVsX3NpZ21hKSkgYW5kIHJlZHVjZQogIGRlbnNpdHkgPC0gc3VtKGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkpCiAgcmV0dXJuKGRlbnNpdHkpCn0KIyBHZXQgcGVyLXByZWRpY3Rpb24gbG9nLWxpa2VsaWhvb2QKbG9nTGlrX3Rlc3RfcGVyIDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpCiAgZGVuc2l0aWVzIDwtIGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkKICByZXR1cm4oZGVuc2l0aWVzKQp9CiMgQ29tcHV0ZSBNU0Ugb2YgYSBuZXcgZGF0YXNldCB1c2luZyBhIGZpdCBsbWU0IG1vZGVsLgptc2VfdGVzdCA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICByZXR1cm4obWVhbigocHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKSAtIHRlc3RfeSkgXiAyKSkKfQojU2FuaXR5IGNoZWNrcwojbXlsbSA8LSBnYW0ocHN5Y2hvbWV0cmljIH4gIHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSwgZGF0YT10cmFpbl9kYXRhKQojYyhsb2dMaWsobXlsbSksIGxvZ0xpa190ZXN0KG15bG0sIHRyYWluX2RhdGEsIHRyYWluX2RhdGEkcHN5Y2hvbWV0cmljKSkKI2xvZ0xpa190ZXN0KG15bG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKYGBgCgojIERhdGEgbG9hZGluZyBhbmQgcHJlcHJvY2Vzc2luZwoKYGBge3IgTG9hZCBhbmQgcHJlcHJvY2VzcyBkYXRhfQpkYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvaGFybW9uaXplZF9yZXN1bHRzLmNzdiIpCgphbGxfZGF0YSA9IGRhdGEgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIGdyb3VwX2J5KGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUKICAgIG11dGF0ZShwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgcHJldl9jb2RlID0gbGFnKGNvZGUpLAogICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgIHByZXZfc3VycCA9IGxhZyhzdXJwcmlzYWwpLAogICAgICAgICAKICAgICAgICAgcHJldjJfZnJlcSA9IGxhZyhwcmV2X2ZyZXEpLAogICAgICAgICBwcmV2Ml9jb2RlID0gbGFnKHByZXZfY29kZSksCiAgICAgICAgIHByZXYyX2xlbiA9IGxhZyhwcmV2X2xlbiksCiAgICAgICAgIHByZXYyX3N1cnAgPSBsYWcocHJldl9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXYzX2ZyZXEgPSBsYWcocHJldjJfZnJlcSksCiAgICAgICAgIHByZXYzX2NvZGUgPSBsYWcocHJldjJfY29kZSksCiAgICAgICAgIHByZXYzX2xlbiA9IGxhZyhwcmV2Ml9sZW4pLAogICAgICAgICBwcmV2M19zdXJwID0gbGFnKHByZXYyX3N1cnApLAogICAgICAgICAKICAgICAgICAgcHJldjRfZnJlcSA9IGxhZyhwcmV2M19mcmVxKSwKICAgICAgICAgcHJldjRfY29kZSA9IGxhZyhwcmV2M19jb2RlKSwKICAgICAgICAgcHJldjRfbGVuID0gbGFnKHByZXYzX2xlbiksCiAgICAgICAgIHByZXY0X3N1cnAgPSBsYWcocHJldjNfc3VycCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAKICAjIEZpbHRlciBiYWNrIHR3byBmb3IgdGhlIGR1bmRlZSBjb3JwdXMuIEZpbHRlciBiYWNrIDEgZm9yIGFsbCBvdGhlciBjb3Jwb3JhCiAgIyBOQiB0aGlzIGVmZmVjdGl2ZWx5IHJlbW92ZXMgYWxsIHplcm8tc3VycHJpc2FsIHJvd3MsIHNpbmNlIGVhcmx5LXNlbnRlbmNlIHRva2VucyBkb24ndCBoYXZlIGNvbnRpZ3VvdXMgdG9rZW4gaGlzdG9yeQogIGZpbHRlcigoY29ycHVzID09ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2Ml9jb2RlICsgMikgfCAoY29ycHVzICE9ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2NF9jb2RlICsgNCkpICU+JQogIAogIHNlbGVjdCgtcHJldl9jb2RlLCAtcHJldjJfY29kZSwgLXByZXYzX2NvZGUpICU+JQogIGRyb3BfbmEoKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUKICBtdXRhdGUoCiAgICBtb2RlbCA9IGFzLmNoYXJhY3Rlcihtb2RlbCksCiAgICBtb2RlbCA9IGlmX2Vsc2UobW9kZWwgPT0gImdwdC0yIiwgImdwdDIiLCBtb2RlbCksCiAgICBtb2RlbCA9IGFzLmZhY3Rvcihtb2RlbCkpCmBgYAoKYGBge3J9Cm1pc3Npbmdfcm93cyA9IGFsbF9kYXRhICU+JSBjb21wbGV0ZShuZXN0aW5nKGNvcnB1cywgY29kZSksIG5lc3RpbmcobW9kZWwsIHRyYWluaW5nLCBzZWVkKSkgJT4lIAogIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIAogICAgZmlsdGVyKHN1bShpcy5uYShzdXJwcmlzYWwpKSA+IDApICU+JSAKICB1bmdyb3VwKCkgJT4lIAogIGFudGlfam9pbihhbGxfZGF0YSwgYnk9YygiY29ycHVzIiwgImNvZGUiLCAibW9kZWwiLCAidHJhaW5pbmciLCAic2VlZCIpKQoKbWlzc2luZ19yb3dzICU+JSBnZ3Bsb3QoYWVzKHg9Y29ycHVzLCBmaWxsPWZhY3RvcihwYXN0ZShtb2RlbCx0cmFpbmluZykpKSkgKyBnZW9tX2Jhcihwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjgpKQpwcmludChtaXNzaW5nX3Jvd3MgJT4lIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUgc3VtbWFyaXNlKG49bigpKSkgJT4lIGFycmFuZ2UoZGVzYyhuKSkKYGBgCgoKYGBge3IgRHJvcCB0b2tlbnMgZm9yIHdoaWNoIGFueSBtb2RlbCBpcyBtaXNzaW5nIHN1cnByaXNhbCBkYXRhLn0KCiMgQ29tcHV0ZSB0aGUgaWRlYWwgbnVtYmVyIG9mIG1vZGVsLS1zZWVkLS10cmFpbmluZyBvYnNlcnZhdGlvbnMgcGVyIHRva2VuLgp0b19kcm9wID0gYWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKG4gPSBuKCkpICU+JSB1bmdyb3VwKCkgJT4lCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUgbXV0YXRlKCBtYXhfbiA9IG1heChuKSkgJT4lIHVuZ3JvdXAoKSAlPiUKICBmaWx0ZXIobWF4X24gIT0gbikgJT4lCiAgc2VsZWN0KGNvZGUsIGNvcnB1cykKCiN0b19kcm9wID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihuKCkgIT0gaWRlYWxfdG9rZW5fb2JzX2NvdW50KSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUgYW50aV9qb2luKHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSksIGJ5PWMoImNvcnB1cyIsICJjb2RlIikpCmxvZ2luZm8ocGFzdGUoIkFmdGVyIGRyb3AsIiwgbnJvdyhhbGxfZGF0YSksICJvYnNlcnZhdGlvbnMgKCIsIGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAiIHRva2VucykgcmVtYWluLiIpKQpgYGAKCmBgYHtyIERyb3AgdG9rZW5zIGZvciB3aGljaCBhbnkgbW9kZWwgaGFzIHplcm8tdmFsdWVkIHN1cnByaXNhbHMufQoKdG9fZHJvcF96ZXJvX3N1cnBzID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihhbnkoc3VycHJpc2FsID09IDApKSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wX3plcm9fc3VycHMpLCAib2JzZXJ2YXRpb25zIGNvcnJlc3BvbmRpbmcgdG8gY29ycHVzIHRva2VucyB3aGljaCBoYXZlIHN1cnByaXNhbCB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCB0b19kcm9wX3plcm9fc3VycHMgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggaGF2ZSBzdXJwcmlzYWwgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JSBhbnRpX2pvaW4odG9fZHJvcF96ZXJvX3N1cnBzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpKQpsb2dpbmZvKHBhc3RlKCJBZnRlciBkcm9wLCIsIG5yb3coYWxsX2RhdGEpLCAib2JzZXJ2YXRpb25zICgiLCBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgIiB0b2tlbnMpIHJlbWFpbi4iKSkKYGBgCgpgYGB7ciBEcm9wIHRva2VucyBmb3Igd2hpY2ggd2UgaGF2ZSB6ZXJvLXZhbHVlZCBwc3ljaG9tZXRyaWMgZGF0YS59Cgp0b19kcm9wX3plcm9fcHN5Y2hzID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihhbnkocHN5Y2hvbWV0cmljID09IDApKSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wX3plcm9fcHN5Y2hzKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggaGF2ZSBwc3ljaG9tZXRyaWMgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgdG9fZHJvcF96ZXJvX3BzeWNocyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgInRva2VucyB3aGljaCBoYXZlIHBzeWNob21ldHJpYyB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lIGFudGlfam9pbih0b19kcm9wX3plcm9fcHN5Y2hzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpLCBieT1jKCJjb3JwdXMiLCAiY29kZSIpKQpsb2dpbmZvKHBhc3RlKCJBZnRlciBkcm9wLCIsIG5yb3coYWxsX2RhdGEpLCAib2JzZXJ2YXRpb25zICgiLCBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgIiB0b2tlbnMpIHJlbWFpbi4iKSkKYGBgCgogCiMgTGVhcm4gbW9kZWxzCiAKYGBge3J9CiMgQ29tcHV0ZSBsaW5lYXIgbW9kZWwgc3RhdHMgZm9yIHRoZSBnaXZlbiB0cmFpbmluZyBkYXRhIHN1YnNldCBhbmQgZnVsbCB0ZXN0IGRhdGEuCiMgQXV0b21hdGljYWxseSBzdWJzZXRzIHRoZSB0ZXN0IGRhdGEgdG8gbWF0Y2ggdGhlIHJlbGV2YW50IGdyb3VwIGZvciB3aGljaCB3ZSBhcmUgdHJhaW5pbmcgYSBsaW5lYXIgbW9kZWwuCmdldF9sbV9kYXRhIDwtIGZ1bmN0aW9uKGRmLCB0ZXN0X2RhdGEsIGZvcm11bGEsIGZvbGQsIHN0b3JlX2VudikgewogICN0aGlzX2xtIDwtIGdhbShmb3JtdWxhLCBkYXRhPWRmKTsKICB0aGlzX2xtID0gbG0oZm9ybXVsYSwgZGF0YT1kZikKICB0aGlzX3Rlc3RfZGF0YSA8LSBzZW1pX2pvaW4odGVzdF9kYXRhLCBkZiwgYnk9YygidHJhaW5pbmciLCAibW9kZWwiLCAic2VlZCIsICJjb3JwdXMiKSk7CiAgCiAgIyBTYXZlIGxtIHRvIHRoZSBnbG9iYWwgZW52IHNvIHRoYXQgd2UgY2FuIGFjY2VzcyByZXNpZHVhbHMgbGF0ZXIuCiAgbG1fbmFtZSA9IHBhc3RlKHVuaXF1ZShwYXN0ZShkZiRtb2RlbCwgZGYkdHJhaW5pbmcsIGRmJHNlZWQsIGRmJGNvcnB1cykpWzFdLCBmb2xkKQogIGFzc2lnbihsbV9uYW1lLCB0aGlzX2xtLCBlbnZpcj1zdG9yZV9lbnYpCiAgCiAgc3VtbWFyaXNlKGRmLAogICAgICAgICAgICBsb2dfbGlrID0gYXMubnVtZXJpYyhsb2dMaWsodGhpc19sbSwgUkVNTCA9IEYpKSwKICAgICAgICAgICAgdGVzdF9saWsgPSBsb2dMaWtfdGVzdCh0aGlzX2xtLCB0aGlzX3Rlc3RfZGF0YSwgdGhpc190ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKSwKICAgICAgICAgICAgdGVzdF9tc2UgPSBtc2VfdGVzdCh0aGlzX2xtLCB0aGlzX3Rlc3RfZGF0YSwgdGhpc190ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKSkKfQojIEZvciBhIHByZXZpb3VzbHkgZml0dGVkIGxtIHN0b3JlZCBpbiBzdG9yZV9lbnYsIGdldCB0aGUgcmVzaWR1YWxzIG9uIHRlc3QgZGF0YSBvZiB0aGUgcmVsZXZhbnQgZGF0YSBzdWJzZXQuCmdldF9sbV9yZXNpZHVhbHMgPC0gZnVuY3Rpb24oZGYsIGZvbGQsIHN0b3JlX2VudikgewogICMgUmV0cmlldmUgdGhlIHJlbGV2YW50IGxtLgogIGxtX25hbWUgPSBwYXN0ZSh1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXSwgZm9sZCkKICB0aGlzX2xtIDwtIGdldChsbV9uYW1lLCBlbnZpcj1zdG9yZV9lbnYpCiAgCiAgbXV0YXRlKGRmLAogICAgICAgICBsaWtlbGlob29kID0gbG9nTGlrX3Rlc3RfcGVyKHRoaXNfbG0sIGRmLCBkZiRwc3ljaG9tZXRyaWMpLAogICAgICAgICByZXNpZCA9IGRmJHBzeWNob21ldHJpYyAtIHByZWRpY3QodGhpc19sbSwgZGYsIHJlLmZvcm09TkEpKQp9CiMgQ29tcHV0ZSBwZXItZXhhbXBsZSBkZWx0YS1sb2ctbGlrZWxpaG9vZCBmb3IgdGhlIGdpdmVuIHRlc3QgZm9sZC4KZ2V0X2xtX2RlbHRhX2xvZ19saWsgPC0gZnVuY3Rpb24odGVzdF9kYXRhLCBmb2xkLCBiYXNlbGluZV9lbnYsIGZ1bGxfZW52KSB7CiAgbG1fbmFtZSA9IHBhc3RlKHVuaXF1ZShwYXN0ZSh0ZXN0X2RhdGEkbW9kZWwsIHRlc3RfZGF0YSR0cmFpbmluZywgdGVzdF9kYXRhJHNlZWQsIHRlc3RfZGF0YSRjb3JwdXMpKVsxXSwgZm9sZCkKICBiYXNlbGluZV9sbSA8LSBnZXQobG1fbmFtZSwgZW52aXI9YmFzZWxpbmVfZW52KQogIGZ1bGxfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPWZ1bGxfZW52KQogIAogIGRlbHRhX2xvZ19saWsgPSBsb2dMaWtfdGVzdF9wZXIoZnVsbF9sbSwgdGVzdF9kYXRhLCB0ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKSAtIGxvZ0xpa190ZXN0X3BlcihiYXNlbGluZV9sbSwgdGVzdF9kYXRhLCB0ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKQogIHJldHVybihjYmluZCh0ZXN0X2RhdGEsIGRlbHRhX2xvZ19saWs9ZGVsdGFfbG9nX2xpaykpCn0KIyMjIyMKIyBEZWZpbmUgcmVncmVzc2lvbiBmb3JtdWxhZS4KIyBFeWUtdHJhY2tpbmcgcmVncmVzc2lvbjogb25seSB1c2Ugc3VycHJpc2FsIGFuZCBwcmV2aW91cyBzdXJwcmlzYWw7IFNQUlQgcmVncmVzc2lvbjogdXNlIDItYmFjayBmZWF0dXJlcy4KCiNiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKQojYmFzZWxpZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjNfZnJlcSwgcHJldjNfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjRfZnJlcSwgcHJldjRfbGVuLCBicyA9ICJjciIpCgojZnVsbF9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYyX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gImNyIikKI2Z1bGxfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYyX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldjNfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2NF9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjNfZnJlcSwgcHJldjNfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjRfZnJlcSwgcHJldjRfbGVuLCBicyA9ICJjciIpCgpiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbgpiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIHByZXYzX2ZyZXEgKyBwcmV2NF9mcmVxICsgbGVuICsgcHJldl9sZW4gKyBwcmV2Ml9sZW4gKyBwcmV2M19sZW4gKyBwcmV2NF9sZW4KCmZ1bGxfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gc3VycHJpc2FsICsgcHJldl9zdXJwICsgcHJldjJfc3VycCArIHByZXYzX3N1cnAgKyBwcmV2NF9zdXJwICsgZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBwcmV2M19mcmVxICsgcHJldjRfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuICsgcHJldjNfbGVuICsgcHJldjRfbGVuCmZ1bGxfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHN1cnByaXNhbCArIHByZXZfc3VycCArIHByZXYyX3N1cnAgKyBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuCiAgCiMjIyMjCiMgUHJlcGFyZSBmcmFtZXMvZW52aXJvbm1lbnRzIGZvciBzdG9yaW5nIHJlc3VsdHMvb2JqZWN0cy4KYmFzZWxpbmVfcmVzdWx0cyA9IGRhdGEuZnJhbWUoKQpmdWxsX21vZGVsX3Jlc3VsdHMgPSBkYXRhLmZyYW1lKCkKYmFzZWxpbmVfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmZ1bGxfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmxvZ19saWtfZGVsdGFzID0gZGF0YS5mcmFtZSgpCgojUmFuZG9tbHkgc2h1ZmZsZSB0aGUgZGF0YQphbGxfZGF0YTwtYWxsX2RhdGFbc2FtcGxlKG5yb3coYWxsX2RhdGEpKSxdCiNDcmVhdGUgSyBlcXVhbGx5IHNpemUgZm9sZHMKSyA9IDEwCmZvbGRzIDwtIGN1dChzZXEoMSxucm93KGFsbF9kYXRhKSksYnJlYWtzPUssbGFiZWxzPUZBTFNFKQojUGVyZm9ybSAxMCBmb2xkIGNyb3NzIHZhbGlkYXRpb24KCiMgRml0IG1vZGVscyBmb3Igc29tZSBmb2xkIG9mIHRoZSBkYXRhLgpiYXNlbGluZV9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGZvbGQsIGVudikgewogIGlmKGNvcnB1cyA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgYmFzZWxpbmVfcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0gZWxzZSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9Cn0KZnVsbF9tb2RlbF9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGZvbGQsIGVudikgewogIGlmKGNvcnB1c1sxXSA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgZnVsbF9ydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfSBlbHNlIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGZ1bGxfc3BydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfQp9CgojIFByZXBhcmUgYSBuZXcgRW52aXJvbm1lbnQgaW4gd2hpY2ggd2Ugc3RvcmUgZml0dGVkIExNcywgd2hpY2ggd2UnbGwgcXVlcnkgbGF0ZXIgZm9yIHJlc2lkdWFscyBhbmQgb3RoZXIgbWV0cmljcy4KYmFzZWxpbmVfZW52ID0gbmV3LmVudigpCmZ1bGxfZW52ID0gbmV3LmVudigpCgpmb3IoaSBpbiAxOkspIHsgCiAgI1NlZ2VtZW50IHlvdXIgZGF0YSBieSBmb2xkIHVzaW5nIHRoZSB3aGljaCgpIGZ1bmN0aW9uIAogIHRlc3RJbmRleGVzIDwtIHdoaWNoKGZvbGRzPT1pLCBhcnIuaW5kPVRSVUUpCiAgdGVzdF9kYXRhIDwtIGFsbF9kYXRhW3Rlc3RJbmRleGVzLCBdCiAgdHJhaW5fZGF0YSA8LSBhbGxfZGF0YVstdGVzdEluZGV4ZXMsIF0KICAKICAjIENvbXB1dGUgYSBiYXNlbGluZSBsaW5lYXIgbW9kZWwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1SVC1jb3JwdXMgY29tYmluYXRpb24uCiAgYmFzZWxpbmVzID0gdHJhaW5fZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgcHJpbnQobW9kZWwpICU+JQogICAgICBkbyhiYXNlbGluZV9jb3JwdXModW5pcXVlKC4kY29ycHVzKSwgLiwgdGVzdF9kYXRhLCBpLCBiYXNlbGluZV9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKSAlPiUKICAgIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpLAogICAgICAgICAgIGZvbGQgPSBpKQogIAogIGJhc2VsaW5lX3Jlc3VsdHMgPSByYmluZChiYXNlbGluZV9yZXN1bHRzLCBiYXNlbGluZXMpCiAgCiAgIyBDb21wdXRlIGEgZnVsbCBsaW5lYXIgbW9kZWwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkLVJULWNvcnB1cyBjb21iaW5hdGlvbgogIGZ1bGxfbW9kZWxzID0gdHJhaW5fZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZnVsbF9tb2RlbF9jb3JwdXModW5pcXVlKC4kY29ycHVzKSwgLiwgdGVzdF9kYXRhLCBpLCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCksCiAgICAgICAgICAgZm9sZCA9IGkpCiAgCiAgZnVsbF9tb2RlbF9yZXN1bHRzID0gcmJpbmQoZnVsbF9tb2RlbF9yZXN1bHRzLCBmdWxsX21vZGVscykKICAKICAjIENvbXB1dGUgZGVsdGEtbG9nLWxpa2VsaWhvb2RzCiAgZm9sZF9sb2dfbGlrX2RlbHRhcyA9IHRlc3RfZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZ2V0X2xtX2RlbHRhX2xvZ19saWsoLiwgaSwgYmFzZWxpbmVfZW52LCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGxvZ19saWtfZGVsdGFzID0gcmJpbmQobG9nX2xpa19kZWx0YXMsIGZvbGRfbG9nX2xpa19kZWx0YXMpCiAgCiAgZm9sZF9iYXNlbGluZV9yZXNpZHVhbHMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9yZXNpZHVhbHMoLiwgaSwgYmFzZWxpbmVfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkKCiAgYmFzZWxpbmVfcmVzaWR1YWxzID0gcmJpbmQoYmFzZWxpbmVfcmVzaWR1YWxzLCBmb2xkX2Jhc2VsaW5lX3Jlc2lkdWFscykKCiAgZm9sZF9mdWxsX3Jlc2lkdWFscyA9IHRlc3RfZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZ2V0X2xtX3Jlc2lkdWFscyguLCBpLCBmdWxsX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGZ1bGxfcmVzaWR1YWxzID0gcmJpbmQoZnVsbF9yZXNpZHVhbHMsIGZvbGRfZnVsbF9yZXNpZHVhbHMpCn0KYGBgCgpgYGB7cn0KI3dyaXRlLmNzdihmdWxsX3Jlc2lkdWFscywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZnVsbF9yZXNpZHVhbHMuY3N2IikKI3dyaXRlLmNzdihiYXNlbGluZV9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Jhc2VsaW5lX3Jlc2lkdWFscy5jc3YiKQpgYGAKCmBgYHtyfQptb2RlbF9kZWx0YXMgPSBsb2dfbGlrX2RlbHRhcyAlPiUKICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lIAogIHN1bW1hcmlzZShtZWFuX2RlbHRhX2xvZ19saWsgPSBtZWFuKGRlbHRhX2xvZ19saWspLAogICAgICAgICAgICBzZW1fZGVsdGFfbG9nX2xpayA9IHNkKGRlbHRhX2xvZ19saWspIC8gc3FydChsZW5ndGgoZGVsdGFfbG9nX2xpaykpKQpgYGAKCmBgYHtyfQp3cml0ZS5jc3YoZnVsbF9tb2RlbF9yZXN1bHRzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mdWxsX21vZGVsX3Jlc3VsdC5jc3YiKQp3cml0ZS5jc3YoYmFzZWxpbmVfcmVzdWx0cywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvYmFzZWxpbmVfcmVzdWx0cy5jc3YiKQojZnVsbF9tb2RlbF9yZXN1bHRzID0gcmVhZC5jc3YoIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZmZ1bGxfbW9kZWxfcmVzdWx0cy5jc3YiKQojYmFzZWxpbmVfcmVzdWx0cyA9IHJlYWQuY3N2KCIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2ZiYXNlbGluZV9yZXN1bHRzYi5jc3YiKQpgYGAKCmBgYHtyfQptZXRyaWMgPC0gIs6UTG9nTGlrIgojbWV0cmljIDwtICItzpRNU0UiCgojICMgU2VsZWN0IHRoZSByZWxldmFudCBtZXRyaWMuCm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICAgICMgUmV0cmlldmUgdGhlIGN1cnJlbnQgdGVzdCBtZXRyaWMKICAgIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX2xvZ19saWssCiAgICAgICAgICAgZGVsdGFfdGVzdF9zZW0gPSBzZW1fZGVsdGFfbG9nX2xpaykgJT4lCiAgICAjIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX21zZSwKICAgICMgICAgICAgIGRlbHRhX3Rlc3Rfc2VtID0gc2VtX2RlbHRhX21zZSkKICAgIAogICAgIyBSZW1vdmUgdGhlIHJhdyBtZXRyaWNzLgogICAgc2VsZWN0KC1tZWFuX2RlbHRhX2xvZ19saWssIC1zZW1fZGVsdGFfbG9nX2xpaywKICAgICAgICAgICAjLW1lYW5fZGVsdGFfbXNlLCAtc2VtX2RlbHRhX21zZQogICAgICAgICAgICkKbW9kZWxfZGVsdGFzCmBgYAoKYGBge3J9CiMgU2FuaXR5IGNoZWNrOiB0cmFpbmluZyBvbiB0cmFpbit0ZXN0IGRhdGEgc2hvdWxkIHlpZWxkIGltcHJvdmVkIHBlcmZvcm1hbmNlIG92ZXIgdHJhaW5pbmcgb24ganVzdCB0cmFpbmluZyBkYXRhLiAoV2hlbiBldmFsdWF0aW5nIG9uIHRlc3QgZGF0YS4pCiMgZnVsbF9iYXNlbGluZXMgPSBhbGxfZGF0YSAlPiUKIyAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKIyAgIHN1bW1hcmlzZShiYXNlbGluZV90cmFpbl9hbGxfdGVzdF9saWsgPSBsb2dMaWtfdGVzdChsbShwc3ljaG9tZXRyaWMgfiBsZW4gKyBmcmVxICsgc2VudF9wb3MsIGRhdGE9LiksIHNlbWlfam9pbih0ZXN0X2RhdGEsIC4sIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpLCBzZW1pX2pvaW4odGVzdF9kYXRhLCAuLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKSRwc3ljaG9tZXRyaWMpKSAlPiUKIyAgIHVuZ3JvdXAoKQojIGZ1bGxfYmFzZWxpbmVzCiMgCiMgZnVsbF9iYXNlbGluZXMgJT4lCiMgICByaWdodF9qb2luKGJhc2VsaW5lcywgYnk9Yygic2VlZCIsICJ0cmFpbmluZyIsICJtb2RlbCIsICJjb3JwdXMiKSkgJT4lCiMgICBtdXRhdGUoZGVsdGE9YmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrLWJhc2VsaW5lX3Rlc3RfbGlrKSAlPiUKIyAgIHNlbGVjdCgtYmFzZWxpbmVfbGlrKSAjICU+JQojICAgI3NlbGVjdCgtYmFzZWxpbmVfdGVzdF9saWssIC1iYXNlbGluZV90cmFpbl9hbGxfdGVzdF9saWssIC1iYXNlbGluZV9saWssIC1iYXNlbGluZV90ZXN0X21zZSkKYGBgCgojIExvYWQgbGFuZ3VhZ2UgbW9kZWwgZGF0YSAoU3ludGF4R3ltLCBQUEwpCgpgYGB7cn0KbGFuZ3VhZ2VfbW9kZWxfZGF0YSA9IHJlYWQuY3N2KCIuLi9kYXRhL21vZGVsX21ldGFkYXRhLmNzdiIpICU+JQogIG11dGF0ZShtb2RlbCA9IGFzLmNoYXJhY3Rlcihtb2RlbCksCiAgICAgICAgIG1vZGVsID0gaWZfZWxzZShtb2RlbCA9PSAiZ3B0LTIiLCAiZ3B0MiIsIG1vZGVsKSwKICAgICAgICAgbW9kZWwgPSBhcy5mYWN0b3IobW9kZWwpKSAlPiUKICBtdXRhdGUodHJhaW5fc2l6ZSA9IGNhc2Vfd2hlbihzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtbGciKSB+IDQyLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1tZCIpIH4gMTUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLXNtIikgfiA1LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC14cyIpIH4gMSksCiAgICAgICAgIAogICAgICAgICAjIFRyYWluaW5nIHZvY2FidWxhcnkgdXN1YWxseSBjb3ZhcmllcyB3aXRoIHRoZSB0cmFpbmluZyBjb3JwdXMuCiAgICAgICAgICMgQnV0IEJQRSBtb2RlbHMgc2hhcmUgYSB2b2NhYnVsYXJ5IGFjcm9zcyB0cmFpbmluZyBjb3Jwb3JhLgogICAgICAgICB0cmFpbmluZ192b2NhYj1hcy5mYWN0b3IoaWZlbHNlKHN0cl9kZXRlY3QodHJhaW5pbmcsICJncHRicGUiKSwgImdwdGJwZSIsIGFzLmNoYXJhY3Rlcih0cmFpbmluZykpKSwKICAgICAgICAgdHJhaW5pbmdfc291cmNlPWFzLmZhY3RvcihzdHJfcmVwbGFjZShhcy5jaGFyYWN0ZXIodHJhaW5pbmcpLCAiLWdwdGJwZSIsICIiKSkKICAgICAgICAgKSAlPiUKICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSkgJT4lCiAgc2VsZWN0KC1waWQsIC10ZXN0X2xvc3MpICU+JQogIGRpc3RpbmN0KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgLmtlZXBfYWxsID0gVFJVRSkKdGFibGUobGFuZ3VhZ2VfbW9kZWxfZGF0YSRzZWVkKQp0YWJsZShtb2RlbF9kZWx0YXMkc2VlZCkKYGBgCgpGaXJzdCBqb2luIGRlbHRhLW1ldHJpYyBkYXRhIHdpdGggbW9kZWwgYXV4aWxpYXJ5IGRhdGEuCgpgYGB7cn0KbW9kZWxfZGVsdGFzID0gbW9kZWxfZGVsdGFzICU+JQogIG1lcmdlKGxhbmd1YWdlX21vZGVsX2RhdGEsIGJ5ID0gYygic2VlZCIsICJ0cmFpbmluZyIsICJtb2RlbCIpLCBhbGw9VCkgJT4lCiAgZHJvcF9uYSgpCgptb2RlbF9kZWx0YXMKYGBgCgpBbHNvIGpvaW4gb24gdGhlIG9yaWdpbmFsIGxpbmVhciBtb2RlbCBkYXRhLCByYXRoZXIgdGhhbiBjb2xsYXBzaW5nIHRvIGRlbHRhLW1ldHJpY3MuClRoaXMgd2lsbCBzdXBwb3J0IHJlZ3Jlc3Npb25zIGxhdGVyIG9uIHRoYXQgZG9uJ3QgY29sbGFwc2UgYWNyb3NzIGZvbGRzLgoKCiMgRmluYWwgZGF0YSBwcmVwcm9jZXNzaW5nCgpgYGB7ciBGaWx0ZXIgbW9kZWxzIGFuZC9vciBjb3Jwb3JhfQojIEV4Y2x1ZGUgb3JkZXJlZC1uZXVyb25zIGZyb20gYWxsIGFuYWx5c2VzLgptb2RlbF9kZWx0YXMgPC0gbW9kZWxfZGVsdGFzICU+JQogIGZpbHRlcihtb2RlbCAhPSAib3JkZXJlZC1uZXVyb25zIikKYGBgCgoKIyBWaXN1YWxpemF0aW9ucwoKIyMgVGhlIGJhc2ljcwoKYGBge3IsIGZpZy5jYXA9IkNvcnB1cyBzaXplcyJ9CmFsbF9kYXRhICU+JSBnZ3Bsb3QoYWVzKHg9Y29ycHVzKSkgKyBnZW9tX2JhcigpCnByaW50KGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMpICU+JSBzdW1tYXJpc2Uobj1uKCkpKQpgYGAKCgpgYGB7ciwgZmlnLmNhcD0iV29yZCBmcmVxdWVuY3kgZGlzdHJpYnV0aW9uIGJ5IGNvcnB1cyJ9CmFsbF9kYXRhICU+JSAKICBnZ3Bsb3QoYWVzKHg9ZnJlcSwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCmBgYHtyLCBmaWcuY2FwPSJXb3JkIGxlbmd0aCBkaXN0cmlidXRpb24gYnkgY29ycHVzIn0KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeD1sZW4sIGNvbG9yPWNvcnB1cykpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgpgYGB7ciwgZmlnLmNhcD0iU3VycHJpc2FsIGRpc3RyaWJ1dGlvbiBieSBjb3JwdXMifQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCiMjIFByZWRpY3RpdmUgcG93ZXIgYW5kIFNHCgoKYGBge3IgQnkgbW9kZWx9Cm1vZGVsX2RlbHRhcyAlPiUKICBnZ3Bsb3QoYWVzKHg9c2dfc2NvcmUsIHk9ZGVsdGFfdGVzdF9tZWFuKSkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1kZWx0YV90ZXN0X21lYW4tZGVsdGFfdGVzdF9zZW0sIHltYXg9ZGVsdGFfdGVzdF9tZWFuK2RlbHRhX3Rlc3Rfc2VtKSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMsIGFlcyhjb2xvcj10cmFpbmluZ192b2NhYiwgc2hhcGU9bW9kZWwpKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiU3ludGF4IEdlbmVyYWxpemF0aW9uIFNjb3JlIikgKwogICAgZ2d0aXRsZSgiU3ludGFjdGljIEdlbmVyYWxpemF0aW9uIHZzLiBQcmVkaWN0aXZlIFBvd2VyIikgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgZmFjZXRfZ3JpZCh+Y29ycHVzLCBzY2FsZXM9ImZyZWUiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKI2dnc2F2ZSgiLi9jb2dzY2lfaW1hZ2VzL3NnX2xvZ2xpay5wbmciLGhlaWdodD01LHdpZHRoPTYpCmBgYAoKIyMjIFJlZ3Jlc3Npb24gYW5hbHlzZXMKCldlIGNvbnRyb2wgZm9yIGVmZmVjdHMgb2YgcGVycGxleGl0eSBieSByZWxhdGluZyB0aGUgcmVzaWR1YWxzIG9mIGEgYHBlcmZvcm1hbmNlIH4gUFBMYCByZWdyZXNzaW9uIHRvIFNHIHNjb3JlLgoKYGBge3IgUmVzaWR1YWxpemVkIHJlZ3Jlc3Npb259CiMgUHJlcGFyZSBhIHJlc2lkdWFsaXplZCByZWdyZXNzaW9uIGZvciB4MSBvbnRvIHksIGNvbnRyb2xsaW5nIGZvciB0aGUgZWZmZWN0cyBvZiB4Mi4KZF9yZXNpZCA9IG1vZGVsX2RlbHRhcyAlPiUKICBkcm9wX25hKCkgJT4lCiAgCiAgIyBSZXNpZHVhbGl6ZSBkZWx0YSBtZXRyaWMgdy5yLnQgUFBMIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZAogIGdyb3VwX2J5KGNvcnB1cykgJT4lCiAgICBtdXRhdGUocmVzaWQuZGVsdGEgPSByZXNpZChsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZzp0ZXN0X3BwbCkpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgCiAgIyBSZXNpZHVhbGl6ZSBTRyBzY29yZSB3LnIudC4gUFBMIGZvciBlYWNoIHRyYWluaW5nIHZvY2FidWxhcnkKICBncm91cF9ieSh0cmFpbmluZ192b2NhYikgJT4lCiAgICAjIE5CIG5vIG5lZWQgZm9yIHRyYWluaW5nOnBwbCBpbnRlcmFjdGlvbiwgc2luY2Ugd2UncmUgd2l0aGluLWdyb3VwLgogICAgbXV0YXRlKHJlc2lkLnNnID0gcmVzaWQobG0oc2dfc2NvcmUgfiB0ZXN0X3BwbCkpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgIyBDb21wdXRlIHN1bW1hcnkgc3RhdGlzdGljcyBhY3Jvc3MgbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1jb3JwdXMuCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nX3ZvY2FiLCBjb3JwdXMsIHNlZWQpICU+JQogICAgc3VtbWFyaXNlKHJlc2lkLmRlbHRhLm1lYW4gPSBtZWFuKHJlc2lkLmRlbHRhKSwKICAgICAgICAgICAgICByZXNpZC5kZWx0YS5zZW0gPSBzZChyZXNpZC5kZWx0YSkgLyBzcXJ0KGxlbmd0aChyZXNpZC5kZWx0YSkpLAogICAgICAgICAgICAgIHJlc2lkLnNnLm1lYW4gPSBtZWFuKHJlc2lkLnNnKSwKICAgICAgICAgICAgICByZXNpZC5zZy5zZW0gPSBzZChyZXNpZC5zZykgLyBzcXJ0KGxlbmd0aChyZXNpZC5zZykpKQojIE5vdyBwbG90IHJlc2lkdWFsIHZzIFNHCmRfcmVzaWQgJT4lCiAgI2ZpbHRlcihjb3JwdXMgIT0gImJuYy1icm93biIpICU+JQogIGdncGxvdChhZXMoeD1yZXNpZC5zZy5tZWFuLCB5PXJlc2lkLmRlbHRhLm1lYW4pKSArCiAgICB0aGVtZV9idygpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHhtaW49cmVzaWQuc2cubWVhbiAtIHJlc2lkLnNnLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHhtYXg9cmVzaWQuc2cubWVhbiArIHJlc2lkLnNnLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHltaW49cmVzaWQuZGVsdGEubWVhbiAtIHJlc2lkLmRlbHRhLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHltYXg9cmVzaWQuZGVsdGEubWVhbiArIHJlc2lkLmRlbHRhLnNlbSksIGFscGhhPTAuMykgKwogICAgI2dlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1UKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwsIGNvbG9yPXRyYWluaW5nX3ZvY2FiKSkgKwogICAgeWxhYihwYXN0ZSgiUmVzaWR1YWwiLCBtZXRyaWMpKSArCiAgICB4bGFiKCJSZXNpZHVhbCBTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX3NnLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD0xMSkKYGBgCgoKYGBge3IgU3RlcHdpc2UgcmVncmVzc2lvbn0KZG9fc3RlcHdpc2VfcmVncmVzc2lvbiA9IGZ1bmN0aW9uKGN1cl9jb3JwdXMpIHsKICByZWdyZXNzaW9uX2RhdGEgPSBtb2RlbF9kZWx0YXMgJT4lCiAgICBmaWx0ZXIoY29ycHVzID09IGN1cl9jb3JwdXMpCiAgCiAgcHJpbnQoIi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0iKQogIHByaW50KGN1cl9jb3JwdXMpCiAgCiAgbG0xID0gbG0oZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwsIGRhdGEgPSByZWdyZXNzaW9uX2RhdGEpCiAgbG0yID0gbG0oZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwgKyBzZ19zY29yZSwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBwcmludChhbm92YShsbTEsIGxtMikpCiAgc3VtbWFyeShsbTIpCn0KZG9fc3RlcHdpc2VfcmVncmVzc2lvbigiYm5jLWJyb3duIikKZG9fc3RlcHdpc2VfcmVncmVzc2lvbigiZHVuZGVlIikKZG9fc3RlcHdpc2VfcmVncmVzc2lvbigibmF0dXJhbC1zdG9yaWVzIikKYGBgCgojIyBQcmVkaWN0aXZlIHBvd2VyIGFuZCBwZXJwbGV4aXR5CgpgYGB7cn0KbW9kZWxfZGVsdGFzICU+JQogIG11dGF0ZSh0ZXN0X3BwbCA9IGlmX2Vsc2UodGVzdF9wcGwgPiA1MDAsIDMyOS45LCB0ZXN0X3BwbCkpICU+JQogIGdncGxvdChhZXMoeD10ZXN0X3BwbCwgeT1kZWx0YV90ZXN0X21lYW4sIGNvbG9yPXRyYWluaW5nX3ZvY2FiLCBmaWxsID0gdHJhaW5pbmdfdm9jYWIsIHltaW49MCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZ2VvbV90ZXh0KGFlcyh4PTI3NSwgeT0wLCBsYWJlbCA9IGMoIi8vIikpKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pLCBhbHBoYT0wLjQpICsKICAgICNnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9RikgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9NCwgYWVzKHNoYXBlPW1vZGVsLCBjb2xvciA9IHRyYWluaW5nX3ZvY2FiKSkgKwogICAgeWxhYihtZXRyaWMpICsKICAgIHhsYWIoIlRlc3QgUGVycGxleGl0eSIpICsKICAgICNjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMSwgMTYpKSArCiAgICBnZ3RpdGxlKCJUZXN0IFBlcnBsZXhpdHkgdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgc2NhbGVfc2hhcGVfbWFudWFsKHZhbHVlcyA9IGMoMTYsIDE3LCAxNSwgMTgpKSArCiAgICBzY2FsZV94X2NvbnRpbnVvdXMobGFiZWxzPWMoMCwgNTAsIDEwMCwgMTUwLCAyMDAsIDI1MCwgNTAwICw1NTApLCBicmVha3M9YygwLCA1MCwgMTAwLCAxNTAsIDIwMCwgMjUwLCAzMDAsIDM1MCksIG1pbm9yX2JyZWFrcyA9IE5VTEwpICsKICAgIHNjYWxlX3lfY29udGludW91cyhsaW1pdHMgPSBjKDAsIE5BKSwgZXhwYW5kID0gYygwLDApKSArCiAgICBmYWNldF93cmFwKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgIGNvb3JkX2NhcnRlc2lhbihjbGlwPSJvZmYiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9wcGxfbG9nbGlrLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD0xMSkKCmBgYAoKYGBge3J9CgpkbGxfY29yX3Rlc3QgPSBmdW5jdGlvbihkZil7CiAgZGYgJT4lCiAgICBzdW1tYXJpc2UoCiAgICAgIGNvciA9IGNvci50ZXN0KGRmJGRlbHRhX3Rlc3RfbWVhbiwgZGYkdGVzdF9wcGwpJGVzdGltYXRlLAogICAgICBwID0gY29yLnRlc3QoZGYkZGVsdGFfdGVzdF9tZWFuLCBkZiR0ZXN0X3BwbCkkcC52YWx1ZQogICAgKQp9Cgptb2RlbF9kZWx0YXMgJT4lCiAgZmlsdGVyKG1vZGVsICE9ICI1Z3JhbSIpICU+JQogIGdyb3VwX2J5KHRyYWluaW5nLCBjb3JwdXMpICU+JQogICAgbXV0YXRlKG4gPSBuKCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICBmaWx0ZXIobiA+IDIpICU+JQogIGdyb3VwX2J5KHRyYWluaW5nLCBjb3JwdXMpICU+JQogICAgZG8oeyBkbGxfY29yX3Rlc3QoLikgfSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGFycmFuZ2UoY29ycHVzKQoKYGBgCgojIyBFZmZlY3Qgb2YgdHJhaW5pbmcgZGF0YSBzaXplCgpgYGB7ciBPbiBwcmVkaWN0aXZlIHBvd2VyfQptb2RlbF9kZWx0YXMgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBsb2codHJhaW5fc2l6ZSkpICU+JQogIG11dGF0ZShicGUgPSBpZl9lbHNlKHRyYWluaW5nX3ZvY2FiID09ICJncHRicGUiLCAieWVzIiwgIm5vIiksCiAgICAgICAgIGJwZSA9IGFzLmZhY3RvcihicGUpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJhaW5fc2l6ZSwgeT1kZWx0YV90ZXN0X21lYW4sIGNvbG9yPW1vZGVsKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pLCB3aWR0aCA9IDAuMSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQsIGFscGhhPTAuMikgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MywgYWVzKHNoYXBlPWJwZSkpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJMb2cgTWlsbGlvbiBUcmFpbmluZyBUb2tlbnMiKSArCiAgICBnZ3RpdGxlKCJUcmFpbmluZyBTaXplIHZzLiBQcmVkaWN0aXZlIFBvd2VyIikgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgI3NjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCIjQTQyRUYxIiwgIiMzODk0QzgiKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgbGVnZW5kLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIiwKICAgICAgICAgIGxlZ2VuZC5kaXJlY3Rpb24gPSAiaG9yaXpvbnRhbCIsCiAgICAgICAgICBsZWdlbmQua2V5LndpZHRoID0gdW5pdCgwLjMsImNtIiksCiAgICAgICAgICBsZWdlbmQuc3BhY2luZy54ID0gdW5pdCgwLjEsICdjbScpKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC90cmFpbmluZ19sb2dsaWsucG5nIixoZWlnaHQ9NSx3aWR0aD01KQpgYGAKCmBgYHtyfQoKbW9kZWxfY29yX3Rlc3QgPSBmdW5jdGlvbihkZil7CiAgZGYgJT4lCiAgICBzdW1tYXJpc2UoY29yID0gY29yLnRlc3QoZGYkdHJhaW5fc2l6ZSwgZGYkZGVsdGFfdGVzdF9tZWFuKSRlc3RpbWF0ZSwKICAgICAgICAgICAgICBwID0gY29yLnRlc3QoZGYkdHJhaW5fc2l6ZSwgZGYkZGVsdGFfdGVzdF9tZWFuKSRwLnZhbHVlKQp9Cgptb2RlbF9kZWx0YXMgJT4lCiAgZ3JvdXBfYnkobW9kZWwsIGNvcnB1cykgJT4lCiAgICBkbyh7bW9kZWxfY29yX3Rlc3QoLil9KSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgYXJyYW5nZSgpCgoKYGBgCgpgYGB7ciBPbiBTRyBzY29yZX0KbW9kZWxfZGVsdGFzICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gbG9nKHRyYWluX3NpemUpKSAlPiUKICBtdXRhdGUoYnBlID0gaWZfZWxzZSh0cmFpbmluZ192b2NhYiA9PSAiZ3B0YnBlIiwgInllcyIsICJubyIpLAogICAgICAgICBicGUgPSBhcy5mYWN0b3IoYnBlKSkgJT4lCiAgZ2dwbG90KGFlcyh4PXRyYWluX3NpemUsIHk9c2dfc2NvcmUsIGNvbG9yPW1vZGVsKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC4yKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zLCBhZXMoc2hhcGU9YnBlKSkgKwogICAgeWxhYigiU0cgU0NvcmUiKSArCiAgICB4bGFiKCJMb2cgTWlsbGlvbiBUcmFpbmluZyBUb2tlbnMiKSArCiAgICBnZ3RpdGxlKCJUcmFpbmluZyBTaXplIHZzLiBTRyBTY29yZSIpICsKICAgICNzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiI0E0MkVGMSIsICIjMzg5NEM4IikpICsKICAgICNmYWNldF9ncmlkKH5tb2RlbCwgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgbGVnZW5kLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTgpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIiwKICAgICAgICAgIGxlZ2VuZC5kaXJlY3Rpb24gPSAiaG9yaXpvbnRhbCIsCiAgICAgICAgICBsZWdlbmQua2V5LndpZHRoID0gdW5pdCgwLjMsImNtIiksCiAgICAgICAgICBsZWdlbmQuc3BhY2luZy54ID0gdW5pdCgwLjEsICdjbScpKQoKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvdHJhaW5pbmdfc2cucG5nIixoZWlnaHQ9NSx3aWR0aD00KQpgYGAKCgpgYGB7cn0KCm1vZGVsX2Nvcl90ZXN0ID0gZnVuY3Rpb24oZGYpewogIGRmICU+JQogICAgc3VtbWFyaXNlKGNvciA9IGNvci50ZXN0KGRmJHRyYWluX3NpemUsIGRmJHNnX3Njb3JlKSRlc3RpbWF0ZSwKICAgICAgICAgICAgICBwID0gY29yLnRlc3QoZGYkdHJhaW5fc2l6ZSwgZGYkc2dfc2NvcmUpJHAudmFsdWUpCn0KCm1vZGVsX2RlbHRhcyAlPiUKICBncm91cF9ieShtb2RlbCkgJT4lCiAgICBkbyh7bW9kZWxfY29yX3Rlc3QoLil9KSAlPiUKICB1bmdyb3VwKCkKCgoKYGBgCgojIyBTbWl0aCAmIExldnkgcmVwcm9kdWN0aW9uCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCkpICsKICB0aGVtZV9idygpICsKICBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKSArCiAgY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKDAsIDIxKSkgKwogIHRoZW1lKHBhbmVsLnNwYWNpbmcgPSB1bml0KDIuNSwgImNtIikpCmdnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3N1cnBfY29ycl9tYXJnaW5hbHMucG5nIixoZWlnaHQ9MS41LHdpZHRoPTExKQoKYGBgCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihtb2RlbCA9PSAiZ3B0MiIpICU+JQogICNmaWx0ZXIoc3VycHJpc2FsIDwgMTUsIHN1cnByaXNhbCA+IDApICU+JQogIGZpbHRlcihzdXJwcmlzYWw8MjEpICU+JQogIG11dGF0ZShicGU9c3RyX2RldGVjdCh0cmFpbmluZywgImJwZSIpLAogICAgICAgICB0cmFpbmluZ19zb3VyY2U9c3RyX3JlcGxhY2UodHJhaW5pbmcsICItZ3B0YnBlIiwgIiIpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCwgeT1wc3ljaG9tZXRyaWMsIGNvbG9yPXRyYWluaW5nX3NvdXJjZSwgbGluZXR5cGU9YnBlKSkgKwogICAgdGhlbWVfYncoKSArCiAgICBzdGF0X3Ntb290aChzZT1ULCBhbHBoYT0wLjUpICsKICAgICNnZW9tX2Vycm9yYmFyKGNvbG9yPSJibGFjayIsIHdpZHRoPS4yLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0uOSksIGFscGhhPTAuMykgKwogICAgI2dlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbiIpICsKICAgIGZhY2V0X3dyYXAobW9kZWwgfiBjb3JwdXMsIHNjYWxlcz0iZnJlZSIsIG5jb2w9Mywgc3RyaXAucG9zaXRpb24gPSBjKCJyaWdodCIpKSArCiAgICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1sZy1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbS1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cy1ncHRicGUiPSIjODg4ODg4IikpICsKICAgIGNvb3JkX2NhcnRlc2lhbih4bGltID0gYygwLCAyMSkpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xMCksCiAgICAgICAgICBheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChzaXplID0gMTApLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTApLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTApLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9zdXJwX2NvcnIucG5nIixoZWlnaHQ9Nix3aWR0aD0xMikKYGBgCgoKYGBge3J9Cgpjb3JyX3Rlc3QgPSBmdW5jdGlvbihkZil7CiAgZGYgJT4lCiAgICBzdW1tYXJpc2UoCiAgICAgIGNvciA9IGNvci50ZXN0KGRmJHN1cnByaXNhbCwgZGYkcHN5Y2hvbWV0cmljKSRlc3RpbWF0ZQogICAgKQp9CgphbGxfZGF0YSAlPiUKICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIGNvcnB1cywgc2VlZCkgJT4lCiAgICBkbyh7IGNvciA9IGNvcnJfdGVzdCguKX0pICU+JQogIHVuZ3JvdXAoKQoKYGBgCgoKCiMjIyBJbnZlc3RpZ2F0ZSB2YW5pbGxhCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihzdXJwcmlzYWwgPCAxNSwgc3VycHJpc2FsID4gMCkgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJ2YW5pbGxhIikgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cHN5Y2hvbWV0cmljKSkgKwogICAgI3N0YXRfc21vb3RoKHNlPVQsIGFscGhhPTAuNSkgKwogICAgI2dlb21fZXJyb3JiYXIoY29sb3I9ImJsYWNrIiwgd2lkdGg9LjIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPS45KSwgYWxwaGE9MC4zKSArCiAgICBnZW9tX3BvaW50KGFscGhhPTAuMSkgKyAjc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbjogVmFuaWxsYSIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfnRyYWluaW5nLCBzY2FsZXMgPSAiZnJlZSIpCiAgICAjIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLWxnLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjODg4ODg4IiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20tZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzLWdwdGJwZSI9IiM4ODg4ODgiKSkKYGBgCgpgYGB7ciBUb255IEJsYWlyIGlzIHRoZSBzb3VyY2Ugb2YgdGhhdCByaWdodC1zaWRlIGRpcH0KYWxsX2RhdGEgJT4lIAogIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIiwgc3VycHJpc2FsID4gMjAsIHBzeWNob21ldHJpYyA8IDMwMCkKYGBgCgpgYGB7cn0KcHJpbnQoZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIikgJT4lIGFycmFuZ2UoZGVzYyhyZXNpZCkpKQpmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciKSAlPiUgYXJyYW5nZShkZXNjKHJlc2lkKSkgJT4lIGZpbHRlcihyZXNpZCA+IDE1MCkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKCiMjIyBJbnZlc3RpZ2F0ZSBSTk5HCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihzdXJwcmlzYWwgPCAxNSwgc3VycHJpc2FsID4gMCkgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJybm5nIikgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cHN5Y2hvbWV0cmljKSkgKwogICAgI3N0YXRfc21vb3RoKHNlPVQsIGFscGhhPTAuNSkgKwogICAgI2dlb21fZXJyb3JiYXIoY29sb3I9ImJsYWNrIiwgd2lkdGg9LjIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPS45KSwgYWxwaGE9MC4zKSArCiAgICBnZW9tX3BvaW50KGFscGhhPTAuMSkgKyAjc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbjogUk5ORyIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfnRyYWluaW5nLCBzY2FsZXMgPSAiZnJlZSIpCmBgYAoKYGBge3IgVG9ueSBCbGFpciBpcyB0aGUgc291cmNlIG9mIHRoYXQgcmlnaHQtc2lkZSBkaXAgZm9yIFJOTkcgdG9vfQphbGxfZGF0YSAlPiUgCiAgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInJubmciLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciLCBzdXJwcmlzYWwgPiAyMCwgcHN5Y2hvbWV0cmljIDwgMzAwKQpgYGAKCmBgYHtyfQpwcmludChmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInJubmciLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciKSAlPiUgYXJyYW5nZShkZXNjKHJlc2lkKSkpCmZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiLCBtb2RlbCA9PSAicm5uZyIsIHRyYWluaW5nID09ICJibGxpcC1sZyIpICU+JSBhcnJhbmdlKGRlc2MocmVzaWQpKSAlPiUgZmlsdGVyKHJlc2lkID4gMTUwKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCkpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgojIyMgSW52ZXN0aWdhdGUgbmdyYW0gdnMgdmFuaWxsYQoKYGBge3J9Cm5ncmFtX3Jlc2lkcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIobW9kZWwgPT0gIjVncmFtIiwgdHJhaW5pbmcgPT0gImJsbGlwLXNtIikgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIHN1bW1hcmlzZShmcmVxPW1lYW4oZnJlcSksIHBzeWNob21ldHJpYz1tZWFuKHBzeWNob21ldHJpYyksIHN1cnByaXNhbD1tZWFuKHN1cnByaXNhbCksIHJlc2lkPW1lYW4ocmVzaWQpKQp2YW5pbGxhX3Jlc2lkcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIobW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtc20iKSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKGZyZXE9bWVhbihmcmVxKSwgcHN5Y2hvbWV0cmljPW1lYW4ocHN5Y2hvbWV0cmljKSwgc3VycHJpc2FsPW1lYW4oc3VycHJpc2FsKSwgIHJlc2lkPW1lYW4ocmVzaWQpKQpyZXNpZHNfam9pbmVkID0gbmdyYW1fcmVzaWRzICU+JSBsZWZ0X2pvaW4odmFuaWxsYV9yZXNpZHMsIGJ5PWMoImNvcnB1cyIsICJjb2RlIiksIHN1ZmZpeD1jKCIubmdyYW0iLCAiLnZhbmlsbGEiKSkKCnJlc2lkc19qb2luZWQgJT4lIAogIGdncGxvdChhZXMoeD1yZXNpZC5uZ3JhbSwgeT1yZXNpZC52YW5pbGxhKSkgKyBnZW9tX3BvaW50KCkgKyBnZW9tX2FibGluZShzbG9wZT0xLCBjb2xvcj0icmVkIikgKwogIGZhY2V0X2dyaWQofmNvcnB1cykKCnJlc2lkc19qb2luZWQgJT4lIAogIG11dGF0ZShyZXNpZF9kaWZmPXJlc2lkLm5ncmFtIC0gcmVzaWQudmFuaWxsYSkgJT4lIAogIGdncGxvdChhZXMoeD1yZXNpZF9kaWZmKSkgKyBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2RpZmY9YWJzKHJlc2lkLm5ncmFtKSAtIGFicyhyZXNpZC52YW5pbGxhKSwKICAgICAgICAgYmlnPXJlc2lkX2RpZmYgPCAtMTApICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLm5ncmFtLCBjb2xvcj1iaWcpKSArIGdlb21fZGVuc2l0eSgpICsgZmFjZXRfZ3JpZCh+Y29ycHVzKSArCiAgZ2d0aXRsZSgibmdyYW0gc3VycHJpc2FsIG9mIGhpZ2gtaW1wcm92ZW1lbnQgdG9rZW5zIChyZWxhdGl2ZSB0byB2YW5pbGxhKSIpCgpyZXNpZHNfam9pbmVkICU+JSAKICBtdXRhdGUocmVzaWRfYWJzX2RpZmY9YWJzKHJlc2lkLm5ncmFtIC0gcmVzaWQudmFuaWxsYSkpICU+JSAKICBnZ3Bsb3QoYWVzKHg9ZnJlcS5uZ3JhbSwgeT1yZXNpZF9hYnNfZGlmZikpICsgZ2VvbV9wb2ludChhbHBoYT0wLjEpICsgZ2VvbV9zbW9vdGgoKQpgYGAKCiMjIyBJbnZlc3RpZ2F0ZSBncHRicGUgdnMgdmFuaWxsYQoKYGBge3J9CmdwdF9yZXNpZHMgPSBmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKG1vZGVsID09ICJncHQyIiwgdHJhaW5pbmcgPT0gImJsbGlwLXNtLWdwdGJwZSIpICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBzdW1tYXJpc2UoZnJlcT1tZWFuKGZyZXEpLCBwc3ljaG9tZXRyaWM9bWVhbihwc3ljaG9tZXRyaWMpLCBzdXJwcmlzYWw9bWVhbihzdXJwcmlzYWwpLCAgcmVzaWQ9bWVhbihyZXNpZCkpCnZhbmlsbGFfcmVzaWRzID0gZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihtb2RlbCA9PSAidmFuaWxsYSIsIHRyYWluaW5nID09ICJibGxpcC1zbSIpICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBzdW1tYXJpc2UoZnJlcT1tZWFuKGZyZXEpLCBwc3ljaG9tZXRyaWM9bWVhbihwc3ljaG9tZXRyaWMpLCBzdXJwcmlzYWw9bWVhbihzdXJwcmlzYWwpLCAgcmVzaWQ9bWVhbihyZXNpZCkpCnJlc2lkc19qb2luZWQgPSBncHRfcmVzaWRzICU+JSBsZWZ0X2pvaW4odmFuaWxsYV9yZXNpZHMsIGJ5PWMoImNvcnB1cyIsICJjb2RlIiksIHN1ZmZpeD1jKCIuZ3B0IiwgIi52YW5pbGxhIikpCgpyZXNpZHNfam9pbmVkICU+JSAKICBnZ3Bsb3QoYWVzKHg9cmVzaWQuZ3B0LCB5PXJlc2lkLnZhbmlsbGEpKSArIGdlb21fcG9pbnQoKSArIGdlb21fYWJsaW5lKHNsb3BlPTEsIGNvbG9yPSJyZWQiKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2RpZmY9cmVzaWQuZ3B0IC0gcmVzaWQudmFuaWxsYSkgJT4lIAogIGdncGxvdChhZXMoeD1yZXNpZF9kaWZmKSkgKyBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2RpZmY9YWJzKHJlc2lkLmdwdCkgLSBhYnMocmVzaWQudmFuaWxsYSksCiAgICAgICAgIGJpZz1yZXNpZF9kaWZmIDwgLTEwKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbC5ncHQsIGNvbG9yPWJpZykpICsgZ2VvbV9kZW5zaXR5KCkgKyBmYWNldF9ncmlkKH5jb3JwdXMpICsKICBnZ3RpdGxlKCJncHQgc3VycHJpc2FsIG9mIGhpZ2gtaW1wcm92ZW1lbnQgdG9rZW5zIChyZWxhdGl2ZSB0byB2YW5pbGxhKSIpCgpyZXNpZHNfam9pbmVkICU+JSAKICBtdXRhdGUocmVzaWRfYWJzX2RpZmY9YWJzKHJlc2lkLmdwdCAtIHJlc2lkLnZhbmlsbGEpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PWZyZXEuZ3B0LCB5PXJlc2lkX2Fic19kaWZmKSkgKyBnZW9tX3BvaW50KGFscGhhPTAuMSkgKyBnZW9tX3Ntb290aCgpCmBgYAoKIyMjIEludmVzdGlnYXRlIHJlc2lkdWFscyBvdmVyYWxsCgpgYGB7cn0KcmVzaWRfZGVsdGFzID0gZnVsbF9yZXNpZHVhbHMgJT4lIHJpZ2h0X2pvaW4oYmFzZWxpbmVfcmVzaWR1YWxzLCBieT1jKCJjb3JwdXMiLCAiY29kZSIsICJtb2RlbCIsICJ0cmFpbmluZyIsICJzZWVkIiksIHN1ZmZpeD1jKCIuZnVsbCIsICIuYmFzZWxpbmUiKSkgJT4lCiAgc2VsZWN0KHJlc2lkLmJhc2VsaW5lLCByZXNpZC5mdWxsLCBjb2RlLCBzdXJwcmlzYWwuZnVsbCwgcHN5Y2hvbWV0cmljLmZ1bGwsIG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzLCBsZW4uZnVsbCkgJT4lCiAgbXV0YXRlKHJlc2lkLmJhc2VsaW5lLnBvbCA9IGlmX2Vsc2UocmVzaWQuYmFzZWxpbmUgPiAwLCAxLCAwKSwKICAgICAgICAgcmVzaWQuZnVsbC5wb2wgPSBpZl9lbHNlKHJlc2lkLmZ1bGwgPiAwLCAxLCAwKSkgJT4lCiAgbXV0YXRlKHJlc2lkLmJhc2VsaW5lID0gYWJzKHJlc2lkLmJhc2VsaW5lKSwKICAgICAgICAgcmVzaWQuZnVsbCA9IGFicyhyZXNpZC5mdWxsKSkgJT4lCiAgbXV0YXRlKHJlc2lkX2RlbHRhPXJlc2lkLmJhc2VsaW5lIC0gcmVzaWQuZnVsbCwgI3Bvc2l0aXZlIGlzIGJldHRlcgogICAgICAgICB0cmFpbmluZ19zb3VyY2U9YXMuZmFjdG9yKHN0cl9yZXBsYWNlKHRyYWluaW5nLCAiLWdwdGJwZSIsICIiKSksCiAgICAgICAgIGJwZT1zdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIikpCmBgYAoKYGBge3J9CgpyID0gcmVzaWRfZGVsdGFzICU+JQogIGZpbHRlcihyZXNpZC5mdWxsLnBvbCAhPSByZXNpZC5iYXNlbGluZS5wb2wpCgpgYGAKCmBgYHtyfQpyZXNpZF9kZWx0YXMgJT4lCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbC5mdWxsLCB5PXJlc2lkX2RlbHRhLCBjb2xvcj10cmFpbmluZykpICsKICAgIGZhY2V0X2dyaWQobW9kZWx+Y29ycHVzKSArCiAgICBnZW9tX3BvaW50KGFscGhhPTAuMSwgc2l6ZT0wLjUpCmBgYAoKYGBge3J9Cmxhbmd1YWdlX21vZGVsX2RhdGEgJT4lIGZpbHRlcihtb2RlbCA9PSAiZ3B0MiIpCmBgYAoKYGBge3J9CnJlc2lkX2RlbHRhcyAlPiUKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgbXV0YXRlKHBzeWNob21ldHJpYyA9IHNjYWxlKHBzeWNob21ldHJpYy5mdWxsKSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGdncGxvdChhZXMoeD1wc3ljaG9tZXRyaWMpKSArCiAgICB0aGVtZV9idygpICsKICAgIGdlb21fZGVuc2l0eSgpICsKICAgIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IDAsIGNvbG9yID0gImdyZXkiKSArCiAgICBmYWNldF9ncmlkKC5+Y29ycHVzKSArCiAgICAjY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKC0yLCA0KSkgKwogICAgdGhlbWUoYXhpcy50aXRsZS54PWVsZW1lbnRfYmxhbmsoKSwKICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwKICAgICAgICAgIGF4aXMudGlja3MueD1lbGVtZW50X2JsYW5rKCkpCiNnZ3NhdmUoImxlbmd0aC5wbmciLCB3aWR0aCA9IDgsIGhlaWdodCA9IDEpCgpgYGAKCmBgYHtyfQoKbG9nX2xpa19kZWx0YXMgICU+JQojcmVzaWRfZGVsdGFzICU+JQogICNmaWx0ZXIocmVzaWQuZnVsbC5wb2wgPT0gcmVzaWQuYmFzZWxpbmUucG9sKSAlPiUKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgbXV0YXRlKHBzeWNob21ldHJpYyA9IHNjYWxlKHBzeWNob21ldHJpYykpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAjZmlsdGVyKHBzeWNob21ldHJpYyA8IDQpICU+JQogICNmaWx0ZXIobGVuLmZ1bGwgPD0gMTApICU+JQogIGdncGxvdChhZXMoeCA9IHBzeWNob21ldHJpYywgeSA9IGRlbHRhX2xvZ19saWssIGNvbG9yID0gbW9kZWwpKSArCiAgICB0aGVtZV9idygpICsKICAgIGZhY2V0X2dyaWQoLiB+IGNvcnB1cywgc2NhbGVzID0gImZyZWUiKSArCiAgICAjZ2VvbV9ydWcoYWxwaGEgPSAwLjAwMywgc2lkZXMgPSAiYiIpICsKICAgIGdlb21faGxpbmUoeWludGVyY2VwdD0wLCBjb2xvciA9ICJibHVlIikgKwogICAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgY29sb3IgPSAiZ3JleSIpICsKICAgIGdlb21fc21vb3RoKHNlID0gVCwgYWxwaGEgPSAwLjIpICsKICAgIGNvb3JkX2NhcnRlc2lhbih5bGltID0gYygtMC4xLCAwLjIpLCB4bGltID0gYygtMiwgNCkpICsKICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF9ibGFuaygpKSAKZ2dzYXZlKCAiLi9yZXNpZF9wc3ljaG8ucG5nIiwgaGVpZ2h0ID0gNCwgd2lkdGggPSA4KQoKCmBgYAoKYGBge3J9CnJlc2lkX2RlbHRhcyAlPiUKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgbXV0YXRlKHBzeWNob21ldHJpYyA9IHNjYWxlKHBzeWNob21ldHJpYy5mdWxsKSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGdncGxvdChhZXMoeD1sZW4uZnVsbCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZ2VvbV9oaXN0b2dyYW0oYmlucyA9IDIwKSArCiAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBjb2xvciA9ICJncmV5IikgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cykgKwogICAgY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKDEsIDEwKSkgKwogICAgdGhlbWUoYXhpcy50aXRsZS54PWVsZW1lbnRfYmxhbmsoKSwKICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwKICAgICAgICAgIGF4aXMudGlja3MueD1lbGVtZW50X2JsYW5rKCkpCmdnc2F2ZSgibGVuZ3RoX21hcmluZ2Fscy5wbmciLCB3aWR0aCA9IDgsIGhlaWdodCA9IDEpCmBgYAoKYGBge3J9Cgpsb2dfbGlrX2RlbHRhcyAgJT4lCiNyZXNpZF9kZWx0YXMgJT4lCiAgI2ZpbHRlcihyZXNpZC5mdWxsLnBvbCA9PSByZXNpZC5iYXNlbGluZS5wb2wpICU+JQogIGdyb3VwX2J5KGNvcnB1cykgJT4lCiAgICBtdXRhdGUocHN5Y2hvbWV0cmljID0gc2NhbGUocHN5Y2hvbWV0cmljKSkgJT4lCiAgdW5ncm91cCgpICU+JQogICNmaWx0ZXIocHN5Y2hvbWV0cmljIDwgNCkgJT4lCiAgI2ZpbHRlcihsZW4uZnVsbCA8PSAxMCkgJT4lCiAgZ2dwbG90KGFlcyh4ID0gbGVuLCB5ID0gZGVsdGFfbG9nX2xpaywgY29sb3IgPSBtb2RlbCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZmFjZXRfZ3JpZCguIH4gY29ycHVzLCBzY2FsZXMgPSAiZnJlZSIpICsKICAgICNnZW9tX3J1ZyhhbHBoYSA9IDAuMDAzLCBzaWRlcyA9ICJiIikgKwogICAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTAsIGNvbG9yID0gImJsdWUiKSArCiAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBjb2xvciA9ICJncmV5IikgKwogICAgZ2VvbV9zbW9vdGgoc2UgPSBULCBhbHBoYSA9IDAuMikgKwogICAgY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKC0wLjAyLCAwLjA2KSwgeGxpbSA9IGMoMSwgMTApKSArCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIiwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfYmxhbmsoKSkgCmdnc2F2ZSggIi4vcmVzaWRfbGVuZ3RoLnBuZyIsIGhlaWdodCA9IDQsIHdpZHRoID0gOCkKYGBgCgoKYGBge3J9Cgp3b3JkX25vcm0gPSBsb2dfbGlrX2RlbHRhcyAlPiUKICBkcm9wX25hKCkgJT4lCiAgZ3JvdXBfYnkod29yZCwgY29ycHVzLCBtb2RlbCwgdHJhaW5pbmcsIHNlZWQpICU+JSAKICBtdXRhdGUocHN5Y2hvd29yZCA9IHNjYWxlKHBzeWNob21ldHJpYyksCiAgICAgICAgIG5vcm1fc3VycCA9IHNjYWxlKHN1cnByaXNhbCkpCmBgYAoKYGBge3J9CndvcmRfbm9ybSAlPiUKICBnZ3Bsb3QoYWVzKHg9bm9ybV9zdXJwKSkgKwogIGZhY2V0X2dyaWQofmNvcnB1cykgKwogIGdlb21fZGVuc2l0eSgpICsKICBjb29yZF9jYXJ0ZXNpYW4oeGxpbSA9IGMoLTIsIDUpKSArCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgY29sb3IgPSAiZ3JleSIpICsKICB0aGVtZV9idygpICsKICAgIHRoZW1lKGF4aXMudGl0bGUueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRleHQueD1lbGVtZW50X2JsYW5rKCksCiAgICAgICAgICBheGlzLnRpY2tzLng9ZWxlbWVudF9ibGFuaygpKQpnZ3NhdmUoInN1cnBfbWFyaW5nYWxzLnBuZyIsIHdpZHRoID0gOCwgaGVpZ2h0ID0gMSkKCgpgYGAKYGBge3J9Cgp3b3JkX25vcm0gJT4lCiAgZ2dwbG90KGFlcyh4ID0gcHN5Y2hvd29yZCwgeSA9IG5vcm1fc3VycCwgY29sb3IgPSBtb2RlbCkpICsKICAgIHRoZW1lX2J3KCkgKwogICAgZmFjZXRfZ3JpZCguIH4gY29ycHVzLCBzY2FsZXMgPSAiZnJlZSIpICsKICAgICNnZW9tX3J1ZyhhbHBoYSA9IDAuMDAzLCBzaWRlcyA9ICJiIikgKwogICAgZ2VvbV9obGluZSh5aW50ZXJjZXB0PTAsIGNvbG9yID0gImJsdWUiKSArCiAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBjb2xvciA9ICJncmV5IikgKwogICAgZ2VvbV9zbW9vdGgoc2UgPSBULCBhbHBoYSA9IDAuMikgKwogICAgI2Nvb3JkX2NhcnRlc2lhbih5bGltID0gYygtMC4wNSwgMC4xKSwgeGxpbSA9IGMoLTIsIDMpKSArCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikgCiNnZ3NhdmUoICIuL3Jlc2lkX2xlbmd0aC5wbmciLCBoZWlnaHQgPSA0LCB3aWR0aCA9IDgpCgpgYGAKCmBgYHtyfQoKd29yZF9ub3JtICU+JQogIGdncGxvdChhZXMoeCA9IG5vcm1fc3VycCwgeSA9IGRlbHRhX2xvZ19saWssIGNvbG9yID0gbW9kZWwpKSArCiAgICB0aGVtZV9idygpICsKICAgIGZhY2V0X2dyaWQoIC4gfiBjb3JwdXMsIHNjYWxlcyA9ICJmcmVlIikgKwogICAgI2dlb21fcnVnKGFscGhhID0gMC4wMDMsIHNpZGVzID0gImIiKSArCiAgICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQ9MCwgY29sb3IgPSAiYmx1ZSIpICsKICAgIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IDAsIGNvbG9yID0gImdyZXkiKSArCiAgICBnZW9tX3Ntb290aChzZSA9IFQsIGFscGhhID0gMC4yKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTAuMDUsIDAuMDcpLCB4bGltID0gYygtMiwgNSkpICsKICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKSAKZ2dzYXZlKCAiLi9ub3JtX3N1cnAucG5nIiwgaGVpZ2h0ID0gNCwgd2lkdGggPSA4KQoKYGBgCgpgYGB7cn0KCm5ncmFtX2hpZ2hzdXJwID0gd29yZF9ub3JtICU+JQogIHVuZ3JvdXAoKSAlPiUKICBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiLCBub3JtX3N1cnAgPiAyLCBtb2RlbCA9PSAiNWdyYW0iKSAlPiUKICBzZWxlY3QoY29kZSkKCm5ncmFtX2hpZ2hzdXJwID0gbmdyYW1faGlnaHN1cnAkY29kZQoKeiA9IHdvcmRfbm9ybSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgZmlsdGVyKCEgY29kZSAlaW4lIG5ncmFtX2hpZ2hzdXJwKSAlPiUKICBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiKQoKd3JpdGUuY3N2KHosICJuZ3JhbS1hYmxhdGUuY3N2IikKCmBgYAoKCgo=